From 213665ddbc17c8c191363ecd19d52553cd85e259 Mon Sep 17 00:00:00 2001 From: kiwipaulrob Date: Sun, 2 Aug 2026 18:04:33 +1200 Subject: [PATCH 01/34] fix(memos-local-plugin): use POSIX-ERE-compatible pgrep pattern for hermes chat detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #1915 pattern used `(?:\s+\S+)*` — a PCRE non-capturing group. `pgrep -f` on Linux compiles patterns with glibc's POSIX ERE, which has no non-capturing groups, so every call failed with: pgrep: regex error: Invalid preceding regular expression `isHermesChatRunning()` swallows the error and returns `false`, so the daemon viewer was permanently stuck on "disconnected" even with a `hermes chat` session attached. The JS unit test passed because JavaScript RegExp accepts `(?:…)` — the proxy was not faithful for ERE. Replace the non-capturing group with a plain capturing group `(\s+\S+)*`, which is valid in both POSIX ERE and JavaScript RegExp, and update the doc comment to warn that the pattern must stay within ERE. Add a regression test that runs the real `pgrep` binary and asserts it does not exit 2 (regex syntax error), so a PCRE-ism can never silently return. --- .../bridge/hermes-process.ts | 21 +++++++++-------- .../tests/unit/bridge/hermes-process.test.ts | 23 ++++++++++++++++--- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/apps/memos-local-plugin/bridge/hermes-process.ts b/apps/memos-local-plugin/bridge/hermes-process.ts index 6c2384f96..b550a1c2a 100644 --- a/apps/memos-local-plugin/bridge/hermes-process.ts +++ b/apps/memos-local-plugin/bridge/hermes-process.ts @@ -18,22 +18,25 @@ * therefore misses any invocation with a global flag (`--skills`, * `-m`, `--provider`, …) between them. * - * The current pattern is `hermes(?:\s+\S+)*\s+chat\b`: + * The current pattern is `hermes(\s+\S+)*\s+chat\b`: * * • `hermes` — the binary basename. - * • `(?:\s+\S+)*` — any complete argv-style tokens between the + * • `(\s+\S+)*` — any complete argv-style tokens between the * binary and the subcommand. * • `\s+chat\b` — a standalone `chat` token, so it does *not* * match `chatter`, `chat-server`, `--chat-log`, or a flag value * like `--profile=chat`. * * `pgrep -f` on Linux uses glibc's ERE engine, which supports - * `\s`/`\b` as GNU extensions. JavaScript's `RegExp` supports the same - * tokens natively, so this module also exports - * `matchesHermesChatCommandLine()` for unit tests — exercising the - * pattern as a JS regex is a faithful proxy for the pgrep-side - * behaviour without requiring a real Hermes binary or a fork of the - * pgrep process in CI. + * `\s`/`\b` as GNU extensions. ⚠️ The pattern MUST stay within POSIX + * ERE — in particular it must NOT use `(?:…)` non-capturing groups, + * which are PCRE-only: glibc ERE rejects the whole pattern with + * "Invalid preceding regular expression", `pgrep` exits 2, and + * `isHermesChatRunning()` silently reports `false`, leaving the + * viewer stuck on `"disconnected"`. A plain capturing group `(…)` + * is valid in both ERE and JavaScript's `RegExp`, so + * `matchesHermesChatCommandLine()` can still proxy the pattern for + * unit tests without a real Hermes binary or a fork of pgrep in CI. */ // eslint-disable-next-line @typescript-eslint/no-require-imports import * as childProcess from "node:child_process"; @@ -45,7 +48,7 @@ import * as childProcess from "node:child_process"; * string we hand to `pgrep` and confirm we have not silently regressed * back to a literal substring match. */ -export const HERMES_CHAT_PROCESS_PATTERN = "hermes(?:\\s+\\S+)*\\s+chat\\b"; +export const HERMES_CHAT_PROCESS_PATTERN = "hermes(\\s+\\S+)*\\s+chat\\b"; /** * JS-side equivalent of `pgrep -f HERMES_CHAT_PROCESS_PATTERN`. diff --git a/apps/memos-local-plugin/tests/unit/bridge/hermes-process.test.ts b/apps/memos-local-plugin/tests/unit/bridge/hermes-process.test.ts index 2f900c7b0..ddd397d57 100644 --- a/apps/memos-local-plugin/tests/unit/bridge/hermes-process.test.ts +++ b/apps/memos-local-plugin/tests/unit/bridge/hermes-process.test.ts @@ -7,10 +7,14 @@ * subcommand (`hermes --skills memory-routing chat`) was silently * missed and the viewer was stuck on `"disconnected"`. * - * The pattern under test is `hermes(?:\s+\S+)*\s+chat\b` — these cases - * lock in the exact shape of the fix. + * The pattern under test is `hermes(\s+\S+)*\s+chat\b` — these cases + * lock in the exact shape of the fix. It must stay valid POSIX ERE + * (no `(?:…)` groups): glibc ERE rejects non-capturing groups, so a + * PCRE-ism in the pattern makes every `pgrep -f` call fail with a + * regex error and `isHermesChatRunning()` return `false` forever. */ import { describe, expect, it, vi } from "vitest"; +import { spawnSync } from "node:child_process"; import { HERMES_CHAT_PROCESS_PATTERN, @@ -23,7 +27,20 @@ describe("HERMES_CHAT_PROCESS_PATTERN", () => { // If this string ever changes, audit `bridge.cts` callers and the // issue description before adjusting — the constant is the only // surface that fixes the substring-detection bug. - expect(HERMES_CHAT_PROCESS_PATTERN).toBe("hermes(?:\\s+\\S+)*\\s+chat\\b"); + expect(HERMES_CHAT_PROCESS_PATTERN).toBe("hermes(\\s+\\S+)*\\s+chat\\b"); + }); + + it("compiles under glibc POSIX ERE — pgrep must not exit 2 (regex error)", () => { + // Regression for the `(?:…)` non-capturing group: JS RegExp accepts + // it, but glibc ERE (what `pgrep -f` uses on Linux) rejects the + // whole pattern with "Invalid preceding regular expression". Run the + // real binary so a PCRE-ism can never silently sneak back in. + // exit 0 = match, 1 = no match (both fine), 2 = regex syntax error. + const result = spawnSync("pgrep", ["-f", HERMES_CHAT_PROCESS_PATTERN], { + encoding: "utf8", + timeout: 2000, + }); + expect(result.status).not.toBe(2); }); }); From 411406dd98370f41c745a8041ea511bf86f4bfad Mon Sep 17 00:00:00 2001 From: CovD <2643822566@qq.com> Date: Tue, 4 Aug 2026 22:44:41 +0800 Subject: [PATCH 02/34] fix(plugin): archive idle low-eta skills --- .../core/config/defaults.ts | 1 + apps/memos-local-plugin/core/config/schema.ts | 6 ++ .../core/skill/ALGORITHMS.md | 15 +++ apps/memos-local-plugin/core/skill/README.md | 3 +- .../core/skill/lifecycle.ts | 3 +- .../core/skill/subscriber.ts | 46 ++++++++- apps/memos-local-plugin/core/skill/types.ts | 2 + .../core/storage/repos/skills.ts | 26 +++++ .../docs/CONFIG-ADVANCED.md | 1 + .../templates/config.demo.yaml | 1 + .../adapters/openclaw-full-chain.test.ts | 48 +++++++++- .../tests/unit/config/load.test.ts | 18 ++++ .../tests/unit/skill/_helpers.ts | 6 +- .../tests/unit/skill/lifecycle.test.ts | 49 +++++++++- .../tests/unit/skill/subscriber.test.ts | 96 +++++++++++++++++++ .../tests/unit/storage/repos.test.ts | 61 ++++++++++++ 16 files changed, 372 insertions(+), 10 deletions(-) diff --git a/apps/memos-local-plugin/core/config/defaults.ts b/apps/memos-local-plugin/core/config/defaults.ts index 5c9dff305..b00f71dde 100644 --- a/apps/memos-local-plugin/core/config/defaults.ts +++ b/apps/memos-local-plugin/core/config/defaults.ts @@ -240,6 +240,7 @@ export const DEFAULT_CONFIG: ResolvedConfig = { etaDelta: 0.1, archiveEta: 0.1, minEtaForRetrieval: 0.1, + idleArchiveMs: 30 * 24 * 60 * 60 * 1000, }, feedback: { failureThreshold: 3, diff --git a/apps/memos-local-plugin/core/config/schema.ts b/apps/memos-local-plugin/core/config/schema.ts index 8566f90f3..af9df703f 100644 --- a/apps/memos-local-plugin/core/config/schema.ts +++ b/apps/memos-local-plugin/core/config/schema.ts @@ -346,6 +346,12 @@ const AlgorithmSchema = Type.Object({ archiveEta: NumberInRange(0.1, 0, 1), /** Hide Tier-1 skills whose η is below this. Mirrors retrieval.minSkillEta. */ minEtaForRetrieval: NumberInRange(0.1, 0, 1), + /** Archive low-η active skills after this much retrieval inactivity. */ + idleArchiveMs: NumberInRange( + 30 * 24 * 60 * 60 * 1000, + 0, + 365 * 24 * 60 * 60 * 1000, + ), }, { default: {} }), feedback: Type.Object({ /** Raise a burst after this many failures of the same tool in-window. */ diff --git a/apps/memos-local-plugin/core/skill/ALGORITHMS.md b/apps/memos-local-plugin/core/skill/ALGORITHMS.md index 45cabf6ae..404f49a6d 100644 --- a/apps/memos-local-plugin/core/skill/ALGORITHMS.md +++ b/apps/memos-local-plugin/core/skill/ALGORITHMS.md @@ -236,6 +236,21 @@ can't take down a well-trialled skill. If the blend drives η under `retireEta` we still retire; the skill can rehab later via positive signals. +### Idle archive scan + +The existing lifecycle tick also archives an active skill when both +conditions hold: + +``` +η < minEtaForRetrieval +now - (lastUsedAt ?? createdAt) >= idleArchiveMs +``` + +`lastUsedAt` is updated by the existing recorded-use path. A never-used +skill falls back to `createdAt`; unrelated metadata updates therefore do +not reset its idle clock. The scan runs through the orchestrator's normal +flush lifecycle and does not introduce a separate timer. + --- ## 7. Retrieval surface diff --git a/apps/memos-local-plugin/core/skill/README.md b/apps/memos-local-plugin/core/skill/README.md index 51ed11e25..c1154ebf5 100644 --- a/apps/memos-local-plugin/core/skill/README.md +++ b/apps/memos-local-plugin/core/skill/README.md @@ -208,6 +208,7 @@ See `algorithm.skill` in | `etaDelta` | `0.1` | η step per `user.positive`/`user.negative`. | | `retireEta` | `0.25` | η floor; crossing retires. | | `minEtaForRetrieval` | `0.5` | η gate for Tier-1 retrieval + auto-promotion. | +| `idleArchiveMs` | `2592000000` | Archive low-η active skills after 30 days without use. | ## Logging @@ -232,7 +233,7 @@ log (`logs/audit.jsonl`, never deleted) via the `skill` channel. * `tests/unit/skill/crystallize.test.ts` — LLM draft normalization + failures. * `tests/unit/skill/verifier.test.ts` — coverage + resonance checks. * `tests/unit/skill/packager.test.ts` — row shape, invocation guide, embedder failure. -* `tests/unit/skill/lifecycle.test.ts` — trial counter, thumbs, retire on drift. +* `tests/unit/skill/lifecycle.test.ts` — trial counter, thumbs, reward drift, and idle archive decisions. * `tests/unit/skill/events.test.ts` — bus contract. * `tests/unit/skill/skill.integration.test.ts` — end-to-end against real SQLite. * `tests/unit/skill/subscriber.test.ts` — event-driven trigger + runOnce + flush. diff --git a/apps/memos-local-plugin/core/skill/lifecycle.ts b/apps/memos-local-plugin/core/skill/lifecycle.ts index a4eaee04a..db2ed0a56 100644 --- a/apps/memos-local-plugin/core/skill/lifecycle.ts +++ b/apps/memos-local-plugin/core/skill/lifecycle.ts @@ -196,7 +196,8 @@ export function shouldArchiveIdle( now: number, ): boolean { if (skill.status !== "active") return false; - const age = now - skill.updatedAt; + const idleSince = skill.lastUsedAt ?? skill.createdAt; + const age = now - idleSince; if (age < idleMs) return false; return skill.eta < cfg.minEtaForRetrieval; } diff --git a/apps/memos-local-plugin/core/skill/subscriber.ts b/apps/memos-local-plugin/core/skill/subscriber.ts index 67754c311..bffcfd5b3 100644 --- a/apps/memos-local-plugin/core/skill/subscriber.ts +++ b/apps/memos-local-plugin/core/skill/subscriber.ts @@ -25,7 +25,7 @@ import { runSkill, type RunSkillDeps, } from "./skill.js"; -import { shouldPromoteCandidate } from "./lifecycle.js"; +import { shouldArchiveIdle, shouldPromoteCandidate } from "./lifecycle.js"; import type { RunSkillInput, RunSkillResult, @@ -210,12 +210,12 @@ export function attachSkillSubscriber( } } - /** Periodic lifecycle pass: promote eligible candidate skills to active. */ + /** Promote eligible candidates and archive stale low-η active skills. */ async function lifecycleTick(): Promise { + const at = nowMs(); const candidates = deps.repos.skills.list({ status: "candidate", limit: 500 }); for (const s of candidates) { if (!shouldPromoteCandidate(s, deps.config)) continue; - const at = nowMs(); deps.repos.skills.setStatus(s.id, "active", at); log.info("skill.auto_promoted", { skillId: s.id, name: s.name, eta: s.eta }); deps.bus.emit({ @@ -227,6 +227,46 @@ export function attachSkillSubscriber( transition: "promoted", }); } + + const archiveBatchSize = 500; + const cutoff = at - deps.config.idleArchiveMs; + while (true) { + const candidates = deps.repos.skills.listIdleArchiveCandidates({ + minEtaForRetrieval: deps.config.minEtaForRetrieval, + cutoff, + limit: archiveBatchSize, + }); + let archivedThisBatch = 0; + for (const s of candidates) { + if (!shouldArchiveIdle(s, deps.config.idleArchiveMs, deps.config, at)) continue; + deps.repos.skills.setStatus(s.id, "archived", at); + archivedThisBatch += 1; + log.info("skill.idle_archived", { + skillId: s.id, + name: s.name, + eta: s.eta, + lastUsedAt: s.lastUsedAt ?? null, + idleArchiveMs: deps.config.idleArchiveMs, + }); + deps.bus.emit({ + kind: "skill.status.changed", + at, + skillId: s.id, + previous: "active", + next: "archived", + transition: "archived", + }); + } + if (candidates.length < archiveBatchSize) break; + if (archivedThisBatch === 0) { + log.warn("skill.idle_archive_stalled", { + candidateCount: candidates.length, + cutoff, + minEtaForRetrieval: deps.config.minEtaForRetrieval, + }); + break; + } + } } return { dispose, runOnce, applyFeedback, flush, lifecycleTick }; diff --git a/apps/memos-local-plugin/core/skill/types.ts b/apps/memos-local-plugin/core/skill/types.ts index a2799b061..b04e3bc35 100644 --- a/apps/memos-local-plugin/core/skill/types.ts +++ b/apps/memos-local-plugin/core/skill/types.ts @@ -118,6 +118,8 @@ export interface SkillConfig { archiveEta: number; /** Below this η, skills never surface in Tier-1 — matches retrieval config. */ minEtaForRetrieval: number; + /** Archive a low-η active skill after it has not been retrieved for this long. */ + idleArchiveMs: number; } /** diff --git a/apps/memos-local-plugin/core/storage/repos/skills.ts b/apps/memos-local-plugin/core/storage/repos/skills.ts index 2fef2d71e..cb978142d 100644 --- a/apps/memos-local-plugin/core/storage/repos/skills.ts +++ b/apps/memos-local-plugin/core/storage/repos/skills.ts @@ -140,6 +140,32 @@ export function makeSkillsRepo(db: StorageDb) { return db.prepare(sql).all(params).map(mapRow); }, + /** + * Return one oldest-first batch of active skills that already satisfy + * the idle-archive predicate. Filtering in SQLite prevents unrelated + * recently-updated skills from starving older candidates. + */ + listIdleArchiveCandidates(input: { + minEtaForRetrieval: number; + cutoff: number; + limit?: number; + }): SkillRow[] { + const params = { + min_eta: input.minEtaForRetrieval, + cutoff: input.cutoff, + limit: Math.max(1, Math.min(500, Math.floor(input.limit ?? 500))), + }; + const sql = ` + SELECT ${COLUMNS.join(", ")} + FROM skills + WHERE status = 'active' + AND eta < @min_eta + AND COALESCE(last_used_at, created_at) <= @cutoff + ORDER BY COALESCE(last_used_at, created_at) ASC + LIMIT @limit`; + return db.prepare(sql).all(params).map(mapRow); + }, + count(filter: Omit = {}): number { const fragments: string[] = []; const params: Record = {}; diff --git a/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md b/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md index 8cfe175e9..11b80a562 100644 --- a/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md +++ b/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md @@ -156,6 +156,7 @@ algorithm: etaDelta: 0.1 # η step per user.positive/user.negative thumbs archiveEta: 0.25 # η floor; crossing archives minEtaForRetrieval: 0.5 # η gate for Tier-1 retrieval + auto-promotion + idleArchiveMs: 2592000000 # archive low-η skills after 30d without retrieval feedback: failureThreshold: 3 # failures in `failureWindow` that trigger a burst (V7 §6.3) failureWindow: 5 # rolling tool-call window per (toolId, context) diff --git a/apps/memos-local-plugin/templates/config.demo.yaml b/apps/memos-local-plugin/templates/config.demo.yaml index ccaca7012..a2578c121 100644 --- a/apps/memos-local-plugin/templates/config.demo.yaml +++ b/apps/memos-local-plugin/templates/config.demo.yaml @@ -62,3 +62,4 @@ algorithm: minGain: 0.0 candidateTrials: 1 cooldownMs: 0 + idleArchiveMs: 2592000000 # 30 days diff --git a/apps/memos-local-plugin/tests/integration/adapters/openclaw-full-chain.test.ts b/apps/memos-local-plugin/tests/integration/adapters/openclaw-full-chain.test.ts index 6ec654173..fa12b9189 100644 --- a/apps/memos-local-plugin/tests/integration/adapters/openclaw-full-chain.test.ts +++ b/apps/memos-local-plugin/tests/integration/adapters/openclaw-full-chain.test.ts @@ -55,7 +55,7 @@ import { makeTmpDb, type TmpDbHandle } from "../../helpers/tmp-db.js"; import { fakeLlm, type FakeLlmScript } from "../../helpers/fake-llm.js"; import type { LlmClient } from "../../../core/llm/types.js"; import type { EmbedInput, EmbedStats, Embedder } from "../../../core/embedding/types.js"; -import type { EmbeddingVector } from "../../../core/types.js"; +import type { EmbeddingVector, SkillId, SkillRow } from "../../../core/types.js"; import type { AgentKind } from "../../../agent-contract/dto.js"; // ─── Helpers ───────────────────────────────────────────────────────────── @@ -641,4 +641,50 @@ describe("OpenClaw adapter integration — multi-session full V7 chain", () => { JSON.stringify(snapshot, null, 2), ); }); + + it("archives a stale low-η skill when OpenClaw closes its session", async () => { + const thirtyOneDaysMs = 31 * 24 * 60 * 60 * 1_000; + const stale: SkillRow = { + id: "sk_openclaw_idle_archive" as SkillId, + name: "openclaw_idle_archive", + status: "active", + invocationGuide: "# OpenClaw idle archive integration fixture", + procedureJson: null, + eta: 0.05, + support: 3, + gain: 0.05, + trialsAttempted: 0, + trialsPassed: 0, + sourcePolicyIds: [], + sourceWorldModelIds: [], + evidenceAnchors: [], + vec: unitFromSeed("skill:openclaw_idle_archive") as unknown as EmbeddingVector, + createdAt: (NOW - thirtyOneDaysMs) as SkillRow["createdAt"], + updatedAt: NOW as SkillRow["updatedAt"], + lastUsedAt: (NOW - thirtyOneDaysMs) as SkillRow["lastUsedAt"], + version: 1, + }; + db!.repos.skills.upsert(stale); + const bridge = createOpenClawBridge({ + agent: AGENT, + core: core!, + log: { + trace: (_m: string, _c?: unknown) => undefined, + info: (_m: string, _c?: unknown) => undefined, + warn: (_m: string, _c?: unknown) => undefined, + error: (_m: string, _c?: unknown) => undefined, + debug: (_m: string, _c?: unknown) => undefined, + }, + now: () => NOW, + }); + const session = new OpenClawSimulator({ bridge, sessionKey: "s-idle-archive" }); + + await session.turn( + "用 Python 返回字符串 hello", + '```python\ndef hello() -> str:\n return "hello"\n```', + ); + await session.close(); + + expect(db!.repos.skills.getById(stale.id)?.status).toBe("archived"); + }); }); diff --git a/apps/memos-local-plugin/tests/unit/config/load.test.ts b/apps/memos-local-plugin/tests/unit/config/load.test.ts index 5d85fd6e0..759f9fbb5 100644 --- a/apps/memos-local-plugin/tests/unit/config/load.test.ts +++ b/apps/memos-local-plugin/tests/unit/config/load.test.ts @@ -28,6 +28,24 @@ describe("config/loadConfig", () => { expect(cfg.logging.timezone).toBe("America/Los_Angeles"); }); + it("defaults skill idle archival to 30 days and accepts an override", () => { + const thirtyDaysMs = 30 * 24 * 60 * 60 * 1000; + expect(resolveConfig({}).algorithm.skill.idleArchiveMs).toBe(thirtyDaysMs); + expect(resolveConfig({ + algorithm: { skill: { idleArchiveMs: 1_000 } }, + }).algorithm.skill.idleArchiveMs).toBe(1_000); + }); + + it("rejects skill idle archival outside the supported 0-to-365-day range", () => { + const overOneYearMs = 365 * 24 * 60 * 60 * 1000 + 1; + expect(() => resolveConfig({ + algorithm: { skill: { idleArchiveMs: -1 } }, + })).toThrow(/schema validation/); + expect(() => resolveConfig({ + algorithm: { skill: { idleArchiveMs: overOneYearMs } }, + })).toThrow(/schema validation/); + }); + it("rejects invalid logging.timezone with config_invalid", () => { expect(() => resolveConfig({ logging: { timezone: "Not/AZone" } })).toThrow(MemosError); try { diff --git a/apps/memos-local-plugin/tests/unit/skill/_helpers.ts b/apps/memos-local-plugin/tests/unit/skill/_helpers.ts index 549a9f477..9cef40506 100644 --- a/apps/memos-local-plugin/tests/unit/skill/_helpers.ts +++ b/apps/memos-local-plugin/tests/unit/skill/_helpers.ts @@ -40,6 +40,7 @@ export function makeSkillConfig(partial: Partial = {}): SkillConfig etaDelta: 0.1, archiveEta: 0.1, minEtaForRetrieval: 0.1, + idleArchiveMs: 30 * 24 * 60 * 60 * 1000, ...partial, }; } @@ -145,7 +146,9 @@ export interface SeedSkillArgs { trialsPassed?: number; sourcePolicyIds?: readonly PolicyId[]; invocationGuide?: string; + createdAt?: EpochMs; updatedAt?: EpochMs; + lastUsedAt?: EpochMs | null; vec?: EmbeddingVector | null; } @@ -165,8 +168,9 @@ export function seedSkill(handle: TmpDbHandle, args: SeedSkillArgs = {}): SkillR sourceWorldModelIds: [], evidenceAnchors: [], vec: args.vec ?? vec([1, 0, 0]), - createdAt: (args.updatedAt ?? NOW) as SkillRow["createdAt"], + createdAt: (args.createdAt ?? args.updatedAt ?? NOW) as SkillRow["createdAt"], updatedAt: (args.updatedAt ?? NOW) as SkillRow["updatedAt"], + lastUsedAt: args.lastUsedAt ?? null, version: 1, }; handle.repos.skills.upsert(row); diff --git a/apps/memos-local-plugin/tests/unit/skill/lifecycle.test.ts b/apps/memos-local-plugin/tests/unit/skill/lifecycle.test.ts index 86101908e..946cfb663 100644 --- a/apps/memos-local-plugin/tests/unit/skill/lifecycle.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/lifecycle.test.ts @@ -22,6 +22,7 @@ function mkSkill(partial: Partial = {}): SkillRow { vec: null, createdAt: partial.createdAt ?? NOW, updatedAt: partial.updatedAt ?? NOW, + lastUsedAt: partial.lastUsedAt ?? null, version: partial.version ?? 1, }; } @@ -104,9 +105,51 @@ describe("skill/lifecycle", () => { expect(recomputeEta(s, policy, cfg)).toBeCloseTo(0.7, 5); }); - it("shouldArchiveIdle picks up stale active skills with low η", () => { + it("archives a low-η active skill after its last use exceeds idleArchiveMs", () => { + const cfg = makeSkillConfig({ minEtaForRetrieval: 0.6, idleArchiveMs: 1_000 }); + const s = mkSkill({ + status: "active", + eta: 0.4, + lastUsedAt: 1_000 as SkillRow["lastUsedAt"], + }); + expect(shouldArchiveIdle(s, 1_000, cfg, 10_000)).toBe(true); + }); + + it("uses createdAt as the idle baseline for a skill that has never been used", () => { + const cfg = makeSkillConfig({ minEtaForRetrieval: 0.6, idleArchiveMs: 1_000 }); + const s = mkSkill({ + status: "active", + eta: 0.4, + createdAt: 1_000 as SkillRow["createdAt"], + updatedAt: 9_500 as SkillRow["updatedAt"], + lastUsedAt: null, + }); + expect(shouldArchiveIdle(s, 1_000, cfg, 10_000)).toBe(true); + }); + + it("keeps recently used or retrievable active skills", () => { + const cfg = makeSkillConfig({ minEtaForRetrieval: 0.6, idleArchiveMs: 1_000 }); + const recent = mkSkill({ + status: "active", + eta: 0.4, + lastUsedAt: 9_500 as SkillRow["lastUsedAt"], + }); + const retrievable = mkSkill({ + status: "active", + eta: 0.6, + lastUsedAt: 1_000 as SkillRow["lastUsedAt"], + }); + expect(shouldArchiveIdle(recent, 1_000, cfg, 10_000)).toBe(false); + expect(shouldArchiveIdle(retrievable, 1_000, cfg, 10_000)).toBe(false); + }); + + it("archives exactly at the configured idle boundary", () => { const cfg = makeSkillConfig({ minEtaForRetrieval: 0.6 }); - const s = mkSkill({ status: "active", eta: 0.4, updatedAt: 0 as SkillRow["updatedAt"] }); - expect(shouldArchiveIdle(s, 1000, cfg, 10_000)).toBe(true); + const skill = mkSkill({ + status: "active", + eta: 0.4, + lastUsedAt: 9_000 as SkillRow["lastUsedAt"], + }); + expect(shouldArchiveIdle(skill, 1_000, cfg, 10_000)).toBe(true); }); }); diff --git a/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts b/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts index dbda7394c..88ec1eafb 100644 --- a/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts @@ -16,6 +16,7 @@ import { makeSkillConfig, seedPolicy, seedSessionOnly, + seedSkill, seedTrace, } from "./_helpers.js"; @@ -154,4 +155,99 @@ describe("skill/subscriber", () => { expect(r.crystallized).toBe(1); sub.dispose(); }); + + it("archives each stale low-η active skill once without regressing candidate promotion", async () => { + handle = makeTmpDb(); + const h = handle; + const l2Bus = createL2EventBus(); + const rewardBus = createRewardEventBus(); + const bus = createSkillEventBus(); + const events: Array<{ + skillId: string; + previous: string; + next: string; + transition: string; + }> = []; + bus.on("skill.status.changed", (event) => { + if (event.kind !== "skill.status.changed") return; + events.push({ + skillId: event.skillId, + previous: event.previous, + next: event.next, + transition: event.transition, + }); + }); + + const stale = seedSkill(h, { + id: "sk_stale" as never, + name: "stale_skill", + status: "active", + eta: 0.05, + createdAt: 1 as never, + updatedAt: 9_000 as never, + lastUsedAt: 1_000 as never, + }); + const candidate = seedSkill(h, { + id: "sk_candidate" as never, + name: "candidate_skill", + status: "candidate", + eta: 0.7, + createdAt: 1 as never, + updatedAt: 1 as never, + }); + + const sub = attachSkillSubscriber({ + l2Bus, + rewardBus, + bus, + repos: h.repos, + embedder: null, + llm: null, + log: rootLogger.child({ channel: "core.skill.subscriber" }), + config: makeSkillConfig({ minEtaForRetrieval: 0.1, idleArchiveMs: 1_000 }), + }); + + await sub.lifecycleTick(); + await sub.lifecycleTick(); + + expect(h.repos.skills.getById(stale.id)?.status).toBe("archived"); + expect(h.repos.skills.getById(candidate.id)?.status).toBe("active"); + expect(events.filter((event) => event.skillId === stale.id)).toEqual([ + { skillId: stale.id, previous: "active", next: "archived", transition: "archived" }, + ]); + expect(events.filter((event) => event.skillId === candidate.id)).toHaveLength(1); + sub.dispose(); + }); + + it("drains more than one 500-skill idle archive batch in one lifecycle tick", async () => { + handle = makeTmpDb(); + const h = handle; + for (let i = 0; i < 501; i++) { + seedSkill(h, { + id: `sk_stale_${i}` as never, + name: `stale_skill_${i}`, + status: "active", + eta: 0.05, + createdAt: 1 as never, + updatedAt: (i + 1) as never, + lastUsedAt: 1 as never, + }); + } + const sub = attachSkillSubscriber({ + l2Bus: createL2EventBus(), + rewardBus: createRewardEventBus(), + bus: createSkillEventBus(), + repos: h.repos, + embedder: null, + llm: null, + log: rootLogger.child({ channel: "core.skill.subscriber" }), + config: makeSkillConfig({ minEtaForRetrieval: 0.1, idleArchiveMs: 1_000 }), + }); + + await sub.lifecycleTick(); + + expect(h.repos.skills.count({ status: "archived" })).toBe(501); + expect(h.repos.skills.count({ status: "active" })).toBe(0); + sub.dispose(); + }); }); diff --git a/apps/memos-local-plugin/tests/unit/storage/repos.test.ts b/apps/memos-local-plugin/tests/unit/storage/repos.test.ts index 3c859fc19..aef629696 100644 --- a/apps/memos-local-plugin/tests/unit/storage/repos.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/repos.test.ts @@ -314,6 +314,67 @@ describe("storage/repos — happy paths", () => { } }); + it("skills: selects idle archive candidates and excludes a skill after recorded use", () => { + const { repos, cleanup } = makeTmpDb(); + try { + const insertSkill = ( + id: string, + status: "active" | "archived", + eta: number, + createdAt: number, + lastUsedAt: number | null, + ) => { + repos.skills.insert({ + id, + name: id, + status, + invocationGuide: "fixture", + procedureJson: null, + eta, + support: 1, + gain: 0, + trialsAttempted: 0, + trialsPassed: 0, + sourcePolicyIds: [], + sourceWorldModelIds: [], + evidenceAnchors: [], + vec: null, + createdAt, + updatedAt: 10_000, + lastUsedAt, + version: 1, + }); + }; + insertSkill("never_used", "active", 0.05, 50, null); + insertSkill("old_used", "active", 0.05, 1, 100); + insertSkill("recent", "active", 0.05, 1, 9_500); + insertSkill("retrievable", "active", 0.1, 1, 100); + insertSkill("already_archived", "archived", 0.05, 1, 100); + + const candidates = repos.skills.listIdleArchiveCandidates({ + minEtaForRetrieval: 0.1, + cutoff: 9_000, + limit: 500, + }); + expect(candidates.map((skill) => skill.id)).toEqual(["never_used", "old_used"]); + expect(repos.skills.listIdleArchiveCandidates({ + minEtaForRetrieval: 0.1, + cutoff: 9_000, + limit: 1, + }).map((skill) => skill.id)).toEqual(["never_used"]); + + expect(repos.skills.recordUse("old_used", 9_500)).toBe(true); + expect(repos.skills.getById("old_used")?.lastUsedAt).toBe(9_500); + expect(repos.skills.listIdleArchiveCandidates({ + minEtaForRetrieval: 0.1, + cutoff: 9_000, + limit: 500, + }).map((skill) => skill.id)).toEqual(["never_used"]); + } finally { + cleanup(); + } + }); + it("feedback: insert, scoped list, polarity filter", () => { const { repos, cleanup } = makeTmpDb(); try { From c5c0d0cd45ca52c075fa343baed038d51b550bda Mon Sep 17 00:00:00 2001 From: CovD <2643822566@qq.com> Date: Tue, 4 Aug 2026 23:15:13 +0800 Subject: [PATCH 03/34] fix(plugin): address idle archive review --- apps/memos-local-plugin/core/config/schema.ts | 4 ++-- apps/memos-local-plugin/core/skill/ALGORITHMS.md | 4 ++++ apps/memos-local-plugin/core/skill/README.md | 2 +- apps/memos-local-plugin/core/skill/subscriber.ts | 14 +++++++------- .../core/storage/repos/skills.ts | 10 +++++++++- apps/memos-local-plugin/docs/CONFIG-ADVANCED.md | 2 +- .../templates/config.demo.yaml | 2 +- .../tests/unit/config/load.test.ts | 16 ++++++++++++---- 8 files changed, 37 insertions(+), 17 deletions(-) diff --git a/apps/memos-local-plugin/core/config/schema.ts b/apps/memos-local-plugin/core/config/schema.ts index af9df703f..0ac5ade8c 100644 --- a/apps/memos-local-plugin/core/config/schema.ts +++ b/apps/memos-local-plugin/core/config/schema.ts @@ -346,10 +346,10 @@ const AlgorithmSchema = Type.Object({ archiveEta: NumberInRange(0.1, 0, 1), /** Hide Tier-1 skills whose η is below this. Mirrors retrieval.minSkillEta. */ minEtaForRetrieval: NumberInRange(0.1, 0, 1), - /** Archive low-η active skills after this much retrieval inactivity. */ + /** Archive low-η active skills after this much retrieval inactivity (minimum 1 hour). */ idleArchiveMs: NumberInRange( 30 * 24 * 60 * 60 * 1000, - 0, + 60 * 60 * 1000, 365 * 24 * 60 * 60 * 1000, ), }, { default: {} }), diff --git a/apps/memos-local-plugin/core/skill/ALGORITHMS.md b/apps/memos-local-plugin/core/skill/ALGORITHMS.md index 404f49a6d..94daaf7ab 100644 --- a/apps/memos-local-plugin/core/skill/ALGORITHMS.md +++ b/apps/memos-local-plugin/core/skill/ALGORITHMS.md @@ -246,6 +246,10 @@ conditions hold: now - (lastUsedAt ?? createdAt) >= idleArchiveMs ``` +Configuration validation enforces a one-hour minimum for `idleArchiveMs` to +prevent an accidental zero value from archiving every low-η active Skill on +the next lifecycle tick. + `lastUsedAt` is updated by the existing recorded-use path. A never-used skill falls back to `createdAt`; unrelated metadata updates therefore do not reset its idle clock. The scan runs through the orchestrator's normal diff --git a/apps/memos-local-plugin/core/skill/README.md b/apps/memos-local-plugin/core/skill/README.md index c1154ebf5..a0e573b8c 100644 --- a/apps/memos-local-plugin/core/skill/README.md +++ b/apps/memos-local-plugin/core/skill/README.md @@ -208,7 +208,7 @@ See `algorithm.skill` in | `etaDelta` | `0.1` | η step per `user.positive`/`user.negative`. | | `retireEta` | `0.25` | η floor; crossing retires. | | `minEtaForRetrieval` | `0.5` | η gate for Tier-1 retrieval + auto-promotion. | -| `idleArchiveMs` | `2592000000` | Archive low-η active skills after 30 days without use. | +| `idleArchiveMs` | `2592000000` | Archive low-η active skills after 30 days without use (minimum 1 hour). | ## Logging diff --git a/apps/memos-local-plugin/core/skill/subscriber.ts b/apps/memos-local-plugin/core/skill/subscriber.ts index bffcfd5b3..af1b0a57c 100644 --- a/apps/memos-local-plugin/core/skill/subscriber.ts +++ b/apps/memos-local-plugin/core/skill/subscriber.ts @@ -35,6 +35,7 @@ import type { } from "./types.js"; import type { SkillId } from "../types.js"; import { now as nowMs } from "../time.js"; +import { IDLE_ARCHIVE_BATCH_LIMIT } from "../storage/repos/skills.js"; export interface SkillSubscriberDeps extends Omit { @@ -228,16 +229,15 @@ export function attachSkillSubscriber( }); } - const archiveBatchSize = 500; const cutoff = at - deps.config.idleArchiveMs; while (true) { - const candidates = deps.repos.skills.listIdleArchiveCandidates({ + const archiveCandidates = deps.repos.skills.listIdleArchiveCandidates({ minEtaForRetrieval: deps.config.minEtaForRetrieval, cutoff, - limit: archiveBatchSize, + limit: IDLE_ARCHIVE_BATCH_LIMIT, }); let archivedThisBatch = 0; - for (const s of candidates) { + for (const s of archiveCandidates) { if (!shouldArchiveIdle(s, deps.config.idleArchiveMs, deps.config, at)) continue; deps.repos.skills.setStatus(s.id, "archived", at); archivedThisBatch += 1; @@ -257,15 +257,15 @@ export function attachSkillSubscriber( transition: "archived", }); } - if (candidates.length < archiveBatchSize) break; - if (archivedThisBatch === 0) { + if (archiveCandidates.length > 0 && archivedThisBatch === 0) { log.warn("skill.idle_archive_stalled", { - candidateCount: candidates.length, + candidateCount: archiveCandidates.length, cutoff, minEtaForRetrieval: deps.config.minEtaForRetrieval, }); break; } + if (archiveCandidates.length < IDLE_ARCHIVE_BATCH_LIMIT) break; } } diff --git a/apps/memos-local-plugin/core/storage/repos/skills.ts b/apps/memos-local-plugin/core/storage/repos/skills.ts index cb978142d..01b91eba2 100644 --- a/apps/memos-local-plugin/core/storage/repos/skills.ts +++ b/apps/memos-local-plugin/core/storage/repos/skills.ts @@ -14,6 +14,8 @@ import { toJsonText, } from "./_helpers.js"; +export const IDLE_ARCHIVE_BATCH_LIMIT = 500; + const COLUMNS = [ "id", "owner_agent_kind", @@ -153,7 +155,13 @@ export function makeSkillsRepo(db: StorageDb) { const params = { min_eta: input.minEtaForRetrieval, cutoff: input.cutoff, - limit: Math.max(1, Math.min(500, Math.floor(input.limit ?? 500))), + limit: Math.max( + 1, + Math.min( + IDLE_ARCHIVE_BATCH_LIMIT, + Math.floor(input.limit ?? IDLE_ARCHIVE_BATCH_LIMIT), + ), + ), }; const sql = ` SELECT ${COLUMNS.join(", ")} diff --git a/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md b/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md index 11b80a562..82e03230d 100644 --- a/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md +++ b/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md @@ -156,7 +156,7 @@ algorithm: etaDelta: 0.1 # η step per user.positive/user.negative thumbs archiveEta: 0.25 # η floor; crossing archives minEtaForRetrieval: 0.5 # η gate for Tier-1 retrieval + auto-promotion - idleArchiveMs: 2592000000 # archive low-η skills after 30d without retrieval + idleArchiveMs: 2592000000 # archive low-η skills after 30d without retrieval (minimum 1h) feedback: failureThreshold: 3 # failures in `failureWindow` that trigger a burst (V7 §6.3) failureWindow: 5 # rolling tool-call window per (toolId, context) diff --git a/apps/memos-local-plugin/templates/config.demo.yaml b/apps/memos-local-plugin/templates/config.demo.yaml index a2578c121..7359f38bc 100644 --- a/apps/memos-local-plugin/templates/config.demo.yaml +++ b/apps/memos-local-plugin/templates/config.demo.yaml @@ -62,4 +62,4 @@ algorithm: minGain: 0.0 candidateTrials: 1 cooldownMs: 0 - idleArchiveMs: 2592000000 # 30 days + idleArchiveMs: 2592000000 # 30 days; minimum 1 hour diff --git a/apps/memos-local-plugin/tests/unit/config/load.test.ts b/apps/memos-local-plugin/tests/unit/config/load.test.ts index 759f9fbb5..d7f394772 100644 --- a/apps/memos-local-plugin/tests/unit/config/load.test.ts +++ b/apps/memos-local-plugin/tests/unit/config/load.test.ts @@ -30,16 +30,24 @@ describe("config/loadConfig", () => { it("defaults skill idle archival to 30 days and accepts an override", () => { const thirtyDaysMs = 30 * 24 * 60 * 60 * 1000; + const sixHoursMs = 6 * 60 * 60 * 1000; expect(resolveConfig({}).algorithm.skill.idleArchiveMs).toBe(thirtyDaysMs); expect(resolveConfig({ - algorithm: { skill: { idleArchiveMs: 1_000 } }, - }).algorithm.skill.idleArchiveMs).toBe(1_000); + algorithm: { skill: { idleArchiveMs: sixHoursMs } }, + }).algorithm.skill.idleArchiveMs).toBe(sixHoursMs); }); - it("rejects skill idle archival outside the supported 0-to-365-day range", () => { + it("rejects skill idle archival outside the supported one-hour-to-365-day range", () => { + const oneHourMs = 60 * 60 * 1000; const overOneYearMs = 365 * 24 * 60 * 60 * 1000 + 1; + expect(resolveConfig({ + algorithm: { skill: { idleArchiveMs: oneHourMs } }, + }).algorithm.skill.idleArchiveMs).toBe(oneHourMs); + expect(() => resolveConfig({ + algorithm: { skill: { idleArchiveMs: 0 } }, + })).toThrow(/schema validation/); expect(() => resolveConfig({ - algorithm: { skill: { idleArchiveMs: -1 } }, + algorithm: { skill: { idleArchiveMs: oneHourMs - 1 } }, })).toThrow(/schema validation/); expect(() => resolveConfig({ algorithm: { skill: { idleArchiveMs: overOneYearMs } }, From 243b0a63fd79d2f170d3abaf4bf7d33b74a7a97a Mon Sep 17 00:00:00 2001 From: CovD <2643822566@qq.com> Date: Tue, 4 Aug 2026 23:18:42 +0800 Subject: [PATCH 04/34] chore(plugin): align PR with dev-v2.0.29 Remove #2208 changes after maintainers retargeted #2209 from main to dev-v2.0.29. --- .../hermes/memos_provider/__init__.py | 147 +--------- .../hermes/memos_provider/bridge_client.py | 137 +++------ apps/memos-local-plugin/agent-contract/dto.ts | 5 - apps/memos-local-plugin/bridge/methods.ts | 9 - .../core/embedding/embedder.ts | 18 +- .../core/embedding/fetcher.ts | 129 +-------- .../core/embedding/index.ts | 1 - .../core/embedding/providers/cohere.ts | 4 +- .../core/embedding/providers/gemini.ts | 4 +- .../core/embedding/providers/mistral.ts | 4 +- .../core/embedding/providers/openai.ts | 4 +- .../core/embedding/providers/voyage.ts | 4 +- .../core/embedding/retry-worker.ts | 31 +- .../core/embedding/types.ts | 20 +- apps/memos-local-plugin/core/index.ts | 1 - apps/memos-local-plugin/core/llm/client.ts | 11 - apps/memos-local-plugin/core/llm/fetcher.ts | 137 +-------- .../core/llm/providers/anthropic.ts | 4 +- .../core/llm/providers/bedrock.ts | 4 +- .../core/llm/providers/gemini.ts | 4 +- .../core/llm/providers/openai.ts | 4 +- apps/memos-local-plugin/core/llm/types.ts | 9 +- apps/memos-local-plugin/core/pipeline/deps.ts | 29 +- .../core/pipeline/memory-core.ts | 88 +++--- .../core/pipeline/orchestrator.ts | 153 ++-------- .../core/retrieval/llm-filter.ts | 8 - .../core/retrieval/retrieve.ts | 53 +--- .../core/retrieval/types.ts | 6 +- .../core/session/intent-classifier.ts | 8 +- .../core/session/manager.ts | 3 - .../core/session/relation-classifier.ts | 24 +- apps/memos-local-plugin/core/session/types.ts | 2 - .../core/util/foreground-resources.ts | 274 ------------------ .../core/util/rate-limited-llm.ts | 31 +- .../core/util/request-deadline.ts | 37 --- .../core/util/retry-after.ts | 200 ------------- .../memos-local-plugin/core/util/semaphore.ts | 43 +-- .../tests/python/test_bridge_client.py | 165 +---------- .../python/test_hermes_provider_pipeline.py | 69 ----- .../tests/unit/bridge/methods.test.ts | 11 - .../tests/unit/embedding/embedder.test.ts | 59 ---- .../tests/unit/embedding/fetcher.test.ts | 82 ------ .../tests/unit/embedding/retry-worker.test.ts | 60 ---- .../tests/unit/llm/client.test.ts | 39 --- .../tests/unit/llm/fetcher.test.ts | 201 +------------ .../tests/unit/pipeline/memory-core.test.ts | 47 --- .../tests/unit/pipeline/orchestrator.test.ts | 33 --- .../unit/session/intent-classifier.test.ts | 19 -- .../unit/session/relation-classifier.test.ts | 27 -- .../unit/util/foreground-resources.test.ts | 138 --------- .../tests/unit/util/request-deadline.test.ts | 35 --- .../tests/unit/util/retry-after.test.ts | 90 ------ .../tests/unit/util/semaphore.test.ts | 19 -- 53 files changed, 197 insertions(+), 2547 deletions(-) delete mode 100644 apps/memos-local-plugin/core/util/foreground-resources.ts delete mode 100644 apps/memos-local-plugin/core/util/request-deadline.ts delete mode 100644 apps/memos-local-plugin/core/util/retry-after.ts delete mode 100644 apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts delete mode 100644 apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts delete mode 100644 apps/memos-local-plugin/tests/unit/util/retry-after.test.ts delete mode 100644 apps/memos-local-plugin/tests/unit/util/semaphore.test.ts diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index 79b88cbf4..6eeaa000e 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -201,47 +201,6 @@ def _long_rpc_timeout_default() -> float: _LONG_RPC_TIMEOUT = _long_rpc_timeout_default() - -def _prefetch_rpc_timeout_default() -> float: - """Resolve the latency budget for Hermes' foreground memory lookup. - - Hermes places its own short deadline around ``prefetch``. Reusing the - long capture timeout here lets the bridge continue work after the host - has already moved on. Keep a separate, configurable ceiling below the - host's default and forward the corresponding absolute deadline to core. - """ - raw = os.environ.get("MEMOS_HERMES_PREFETCH_RPC_TIMEOUT", "") - try: - value = float(raw) - except (TypeError, ValueError): - return 6.0 - if not value > 0: - return 6.0 - # Hermes currently abandons external providers after 8 seconds. Keep at - # least one second for Python thread scheduling and response assembly even - # when an operator overrides the default. - return min(value, 7.0) - - -_PREFETCH_RPC_TIMEOUT = _prefetch_rpc_timeout_default() -_PREFETCH_RESPONSE_RESERVE_SECONDS = 0.25 - - -def _remaining_rpc_timeout( - deadline_monotonic: float | None, - requested_timeout: float | None, -) -> float | None: - """Bound one blocking bridge step by a shared end-to-end deadline.""" - if deadline_monotonic is None: - return requested_timeout - remaining = deadline_monotonic - time.monotonic() - if remaining <= 0: - raise BridgeError("timeout", "foreground prefetch deadline exceeded") - if requested_timeout is None: - return remaining - return min(requested_timeout, remaining) - - _HERMES_INTERNAL_REVIEW_PREFIXES = ( "review the conversation above and consider saving to memory if appropriate.", "review the conversation above and update the skill library.", @@ -1063,19 +1022,8 @@ def prefetch(self, query: str, *, session_id: str = "") -> str: # type: ignore[ cached result immediately. Otherwise synchronously run ``turn.start`` against the bridge (small overhead). """ - deadline_monotonic = time.monotonic() + _PREFETCH_RPC_TIMEOUT - started_at_ms = int(time.time() * 1000) - core_budget_seconds = max( - 0.05, - _PREFETCH_RPC_TIMEOUT - _PREFETCH_RESPONSE_RESERVE_SECONDS, - ) - deadline_at_ms = started_at_ms + int(core_budget_seconds * 1000) if self._prefetch_thread and self._prefetch_thread.is_alive(): - try: - join_timeout = _remaining_rpc_timeout(deadline_monotonic, 5.0) - except BridgeError: - return "" - self._prefetch_thread.join(timeout=join_timeout) + self._prefetch_thread.join(timeout=5.0) with self._prefetch_lock: cached = self._prefetch_result self._prefetch_result = "" @@ -1085,25 +1033,10 @@ def prefetch(self, query: str, *, session_id: str = "") -> str: # type: ignore[ suppress_injection = _is_explicit_delegation_request(query) if cached: return "" if suppress_injection else cached - try: - ensure_timeout = _remaining_rpc_timeout( - deadline_monotonic, - _PREFETCH_RPC_TIMEOUT, - ) - except BridgeError: - return "" - if not self._ensure_bridge( - session_id or self._session_id, - timeout=min(10.0, ensure_timeout or _PREFETCH_RPC_TIMEOUT), - ): + if not self._ensure_bridge(session_id or self._session_id, timeout=10.0): return "" try: - context = self._turn_start( - query, - session_id=session_id, - deadline_monotonic=deadline_monotonic, - deadline_at_ms=deadline_at_ms, - ) + context = self._turn_start(query, session_id=session_id) if suppress_injection: # Do not let remembered "do it directly" skills override an # explicit user request to dispatch work to a subagent. @@ -2008,7 +1941,6 @@ def _bridge_request( *, timeout: float | None = None, ensure_session: bool = True, - deadline_monotonic: float | None = None, ) -> dict[str, Any]: bridge = self._bridge if bridge is None: @@ -2026,23 +1958,10 @@ def _bridge_request( bridge.generation, self._session_id, ) - session_ceiling = ( - timeout - if deadline_monotonic is not None and timeout is not None - else 30.0 - ) - session_timeout = _remaining_rpc_timeout( - deadline_monotonic, - session_ceiling, - ) - self._open_session( - self._session_id, - timeout=session_timeout or 30.0, - ) - request_timeout = _remaining_rpc_timeout(deadline_monotonic, timeout) - if request_timeout is None: + self._open_session(self._session_id, timeout=30.0) + if timeout is None: return bridge.request(method, params) - return bridge.request(method, params, timeout=request_timeout) + return bridge.request(method, params, timeout=timeout) def _open_session(self, session_id: str = "", *, timeout: float = 30.0) -> None: bridge = self._bridge @@ -2078,7 +1997,6 @@ def _bridge_request_with_retry( params: Any, *, timeout: float | None = None, - deadline_monotonic: float | None = None, ) -> dict[str, Any]: """Read-path helper: reconnect + retry once on ``transport_closed``. @@ -2094,12 +2012,7 @@ def _bridge_request_with_retry( """ assert self._bridge is not None try: - return self._bridge_request( - method, - params, - timeout=timeout, - deadline_monotonic=deadline_monotonic, - ) + return self._bridge_request(method, params, timeout=timeout) except BridgeError as err: if not self._is_transport_closed(err): raise @@ -2108,24 +2021,9 @@ def _bridge_request_with_retry( method, err, ) - reconnect_ceiling = ( - timeout if deadline_monotonic is not None and timeout is not None else 30.0 - ) - reconnect_timeout = _remaining_rpc_timeout( - deadline_monotonic, - reconnect_ceiling, - ) - self._reconnect_bridge( - self._session_id, - timeout=reconnect_timeout or 30.0, - ) + self._reconnect_bridge(self._session_id, timeout=30.0) assert self._bridge is not None - return self._bridge_request( - method, - params, - timeout=timeout, - deadline_monotonic=deadline_monotonic, - ) + return self._bridge_request(method, params, timeout=timeout) def _is_transport_closed(self, err: Exception) -> bool: if isinstance(err, BridgeError) and err.code == "transport_closed": @@ -2335,14 +2233,7 @@ def _run() -> None: ) self._bridge_keepalive_thread.start() - def _turn_start( - self, - query: str, - *, - session_id: str = "", - deadline_monotonic: float | None = None, - deadline_at_ms: int | None = None, - ) -> str: + def _turn_start(self, query: str, *, session_id: str = "") -> str: assert self._bridge is not None host_runtime = self._host_runtime_context() with self._state_lock: @@ -2360,34 +2251,20 @@ def _turn_start( "visibleContextStartTs": visible_context_start_ts, } ) - now_ms = int(time.time() * 1000) - if deadline_monotonic is None: - deadline_monotonic = time.monotonic() + _PREFETCH_RPC_TIMEOUT - if deadline_at_ms is None: - core_budget_seconds = max( - 0.05, - _PREFETCH_RPC_TIMEOUT - _PREFETCH_RESPONSE_RESERVE_SECONDS, - ) - deadline_at_ms = now_ms + int(core_budget_seconds * 1000) payload: dict[str, Any] = { "agent": "hermes", "namespace": self._runtime_namespace(), "sessionId": session_id or self._session_id, "userText": query, "contextHints": context_hints, - "ts": now_ms, - "deadlineAt": deadline_at_ms, + "ts": int(time.time() * 1000), } if turn_key: payload["turnKey"] = turn_key resp = self._bridge_request_with_retry( "turn.start", payload, - timeout=_remaining_rpc_timeout( - deadline_monotonic, - _PREFETCH_RPC_TIMEOUT, - ), - deadline_monotonic=deadline_monotonic, + timeout=_LONG_RPC_TIMEOUT, ) response_query = (resp or {}).get("query") or {} response_session = str(response_query.get("sessionId") or "") diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py index cbb2fa84c..6863ba1ef 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py @@ -17,7 +17,6 @@ import json import logging import os -import queue import shutil import subprocess import threading @@ -33,7 +32,6 @@ logger = logging.getLogger(__name__) HOST_HANDLER_WAIT_SECONDS = 5.0 -HOST_HANDLER_QUEUE_CAPACITY = 16 # ─── Module-level singleton tracker ───────────────────────────────────── # Each entry maps an ``(agent, no_viewer, runtime_home)`` key to the @@ -154,16 +152,12 @@ def __init__( # Reverse-direction handlers: the bridge can send us a # JSON-RPC request via `serverRequest(...)` (e.g. # `host.llm.complete` for fallback LLM calls). Registered - # methods run on one bounded, daemon worker. Keeping execution - # serial preserves the adapter's previous concurrency contract while - # preventing a slow host LLM call from blocking stdout response - # demultiplexing for every shared provider lease. + # methods run on the dedicated reader thread; long-running + # work should spawn its own worker if it needs to. Each + # handler returns a JSON-serialisable value or raises to + # surface a JSON-RPC error back to the bridge. self._host_handlers: dict[str, Callable[[dict[str, Any]], Any]] = {} self._host_handlers_cv = threading.Condition() - self._host_handler_queue: queue.Queue[tuple[Any, str, dict[str, Any]] | None] = queue.Queue( - maxsize=HOST_HANDLER_QUEUE_CAPACITY - ) - self._host_handler_stop = threading.Event() self._closed = False plugin_root = Path(__file__).resolve().parent.parent.parent.parent @@ -232,12 +226,6 @@ def __init__( env=env, cwd=str(plugin_root), ) - self._host_handler_worker = threading.Thread( - target=self._host_handler_loop, - daemon=True, - name="memos-bridge-host-handler", - ) - self._host_handler_worker.start() self._reader = threading.Thread( target=self._read_loop, daemon=True, @@ -360,7 +348,7 @@ def notify(self, method: str, params: Any = None) -> None: try: self._proc.stdin.write(payload + "\n") self._proc.stdin.flush() - except (BrokenPipeError, OSError, ValueError): + except (BrokenPipeError, OSError): pass def on_event(self, cb: Callable[[dict[str, Any]], None]) -> None: @@ -377,9 +365,11 @@ def register_host_handler( """Register a handler for bridge → adapter (reverse) requests. The Node-side bridge calls these via ``stdio.serverRequest``. - Most-recent registration wins. Handlers run serially on a bounded - daemon worker so a long-running host LLM call cannot stall the reader - thread that resolves unrelated foreground JSON-RPC responses. + Most-recent registration wins. The handler runs on the reader + thread; if it blocks for a long time it stalls every other + bridge → adapter notification, so handlers that need to do + heavy work (e.g. an LLM call) are still expected to return + within the bridge-side timeout (default 60 s). """ with self._host_handlers_cv: self._host_handlers[method] = handler @@ -391,7 +381,6 @@ def close(self) -> None: with self._host_handlers_cv: self._closed = True self._host_handlers_cv.notify_all() - self._stop_host_handler_worker() # Drop self from the module-level singleton tracker (issue #1910) # BEFORE the potentially-slow stdin/SIGTERM/SIGKILL dance. We @@ -446,7 +435,6 @@ def _abort_pending(self, reason: str) -> None: with self._host_handlers_cv: self._closed = True self._host_handlers_cv.notify_all() - self._stop_host_handler_worker() with self._lock: for entry in list(self._pending.values()): entry["error"] = { @@ -489,9 +477,7 @@ def _read_loop(self) -> None: # Reverse-direction request: the bridge is asking the # adapter to do something (e.g. run a fallback LLM call # via `host.llm.complete`). Dispatch to the registered - # handler on the bounded worker. The reader must return to - # stdout immediately so a slow host LLM callback cannot - # head-of-line block normal JSON-RPC responses. + # handler and write the response back synchronously. method = msg.get("method") rpc_id = msg.get("id") if ( @@ -500,10 +486,33 @@ def _read_loop(self) -> None: and "result" not in msg and "error" not in msg ): + handler = self._host_handler_for(method) + if handler is None: + self._send_response( + rpc_id, + error={ + "code": -32601, + "message": f"method not found: {method}", + "data": {"code": "unknown_method"}, + }, + ) + continue params = msg.get("params") or {} if not isinstance(params, dict): params = {} - self._dispatch_host_request(rpc_id, method, params) + try: + result = handler(params) + self._send_response(rpc_id, result=result) + except Exception as err: + logger.warning("host handler %s failed: %s", method, err) + self._send_response( + rpc_id, + error={ + "code": -32000, + "message": str(err) or err.__class__.__name__, + "data": {"code": "host_handler_failed"}, + }, + ) continue except Exception: # Any unexpected exception in the reader loop still needs @@ -518,80 +527,6 @@ def _read_loop(self) -> None: # instead of waiting for each 30 s per-request timeout. self._abort_pending("bridge subprocess exited") - def _dispatch_host_request( - self, - rpc_id: Any, - method: str, - params: dict[str, Any], - ) -> None: - """Queue reverse RPC work without ever blocking the reader thread.""" - if self._closed: - return - try: - self._host_handler_queue.put_nowait((rpc_id, method, params)) - except queue.Full: - logger.warning("host handler queue full; rejecting %s", method) - self._send_response( - rpc_id, - error={ - "code": -32000, - "message": "host handler queue is full", - "data": {"code": "host_handler_busy"}, - }, - ) - - def _host_handler_loop(self) -> None: - """Run reverse RPC handlers serially away from stdout demultiplexing.""" - while True: - request = self._host_handler_queue.get() - try: - if request is None: - return - rpc_id, method, params = request - if self._closed: - continue - handler = self._host_handler_for(method) - if handler is None: - self._send_response( - rpc_id, - error={ - "code": -32601, - "message": f"method not found: {method}", - "data": {"code": "unknown_method"}, - }, - ) - continue - try: - result = handler(params) - self._send_response(rpc_id, result=result) - except Exception as err: - logger.warning("host handler %s failed: %s", method, err) - self._send_response( - rpc_id, - error={ - "code": -32000, - "message": str(err) or err.__class__.__name__, - "data": {"code": "host_handler_failed"}, - }, - ) - finally: - self._host_handler_queue.task_done() - - def _stop_host_handler_worker(self) -> None: - """Discard queued callbacks and ask the daemon worker to exit.""" - if self._host_handler_stop.is_set(): - return - self._host_handler_stop.set() - while True: - try: - self._host_handler_queue.get_nowait() - except queue.Empty: - break - else: - self._host_handler_queue.task_done() - with contextlib.suppress(queue.Full): - self._host_handler_queue.put_nowait(None) - def _host_handler_for( self, method: str, @@ -633,7 +568,7 @@ def _send_response( try: self._proc.stdin.write(json.dumps(payload, ensure_ascii=False) + "\n") self._proc.stdin.flush() - except (BrokenPipeError, OSError, ValueError): + except (BrokenPipeError, OSError): pass def _stderr_loop(self) -> None: diff --git a/apps/memos-local-plugin/agent-contract/dto.ts b/apps/memos-local-plugin/agent-contract/dto.ts index b76232bee..e9ae71101 100644 --- a/apps/memos-local-plugin/agent-contract/dto.ts +++ b/apps/memos-local-plugin/agent-contract/dto.ts @@ -103,11 +103,6 @@ export interface TurnInputDTO { contextHints?: Record; /** Wall-clock when the turn began. */ ts: EpochMs; - /** - * Absolute adapter deadline for foreground work. Every pipeline stage - * shares this budget; it is not reset after relation or intent handling. - */ - deadlineAt?: EpochMs; } export interface TurnResultDTO { diff --git a/apps/memos-local-plugin/bridge/methods.ts b/apps/memos-local-plugin/bridge/methods.ts index 1e6cc27a0..2bc1084a6 100644 --- a/apps/memos-local-plugin/bridge/methods.ts +++ b/apps/memos-local-plugin/bridge/methods.ts @@ -389,15 +389,6 @@ function validateTurnInput(p: Record): void { "turn.start: optional 'turnKey' must be a string", ); } - if ( - p.deadlineAt !== undefined && - (typeof p.deadlineAt !== "number" || !Number.isFinite(p.deadlineAt)) - ) { - throw new MemosError( - "invalid_argument", - "turn.start: optional 'deadlineAt' must be a finite number", - ); - } } function validateTurnResult(p: Record): void { diff --git a/apps/memos-local-plugin/core/embedding/embedder.ts b/apps/memos-local-plugin/core/embedding/embedder.ts index 5d035b6ce..bd4a5fe92 100644 --- a/apps/memos-local-plugin/core/embedding/embedder.ts +++ b/apps/memos-local-plugin/core/embedding/embedder.ts @@ -18,7 +18,6 @@ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; import { rootLogger } from "../logger/index.js"; import type { Logger } from "../logger/types.js"; import type { EmbeddingVector } from "../types.js"; -import { extractRetryDiagnostics } from "../util/retry-after.js"; import { LruEmbedCache, NullEmbedCache, @@ -33,7 +32,6 @@ import { MistralEmbeddingProvider } from "./providers/mistral.js"; import { OpenAiEmbeddingProvider } from "./providers/openai.js"; import { VoyageEmbeddingProvider } from "./providers/voyage.js"; import type { - EmbedCallOptions, EmbedInput, EmbedRole, EmbedStats, @@ -89,10 +87,6 @@ export function createEmbedderWithProvider( code?: string; at?: number; durationMs?: number; - retryAfterMs?: number; - retryAt?: number; - retryDecision?: "wait" | "defer" | "stop"; - retryReason?: string; }): void { if (!config.onStatus) return; try { @@ -102,17 +96,13 @@ export function createEmbedderWithProvider( } } - async function embedOne( - input: string | EmbedInput, - options?: EmbedCallOptions, - ): Promise { - const vecs = await embedMany([input], options); + async function embedOne(input: string | EmbedInput): Promise { + const vecs = await embedMany([input]); return vecs[0]!; } async function embedMany( inputs: Array, - options?: EmbedCallOptions, ): Promise { requests += inputs.length; if (inputs.length === 0) return []; @@ -189,8 +179,6 @@ export function createEmbedderWithProvider( const ctx: ProviderCallCtx = { config, log: providerCtxLog, - signal: options?.signal, - deadlineAt: options?.deadlineAt, }; raw = await provider.embed(texts, role, ctx); // Record success but DO NOT clear `lastError` — the viewer @@ -235,7 +223,6 @@ export function createEmbedderWithProvider( message: errMessage, code: err instanceof MemosError ? err.code : undefined, at: errAt, - ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); } catch { /* sink errors are non-fatal */ @@ -249,7 +236,6 @@ export function createEmbedderWithProvider( code: err instanceof MemosError ? err.code : undefined, at: errAt, durationMs: errAt - startedAt, - ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); throw err instanceof MemosError ? err diff --git a/apps/memos-local-plugin/core/embedding/fetcher.ts b/apps/memos-local-plugin/core/embedding/fetcher.ts index 303dae28e..40524aac7 100644 --- a/apps/memos-local-plugin/core/embedding/fetcher.ts +++ b/apps/memos-local-plugin/core/embedding/fetcher.ts @@ -8,15 +8,6 @@ */ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; -import { - getRetryCooldown, - parseRetryAfterMs, - planRetry, - recordRetryCooldown, - retryCooldownKey, - type RetryPlan, - waitForRetry, -} from "../util/retry-after.js"; import type { EmbeddingProviderName, ProviderLogger } from "./types.js"; export interface HttpPostOpts { @@ -26,10 +17,6 @@ export interface HttpPostOpts { timeoutMs?: number; maxRetries?: number; signal?: AbortSignal; - /** Absolute end-to-end deadline. Unlike timeoutMs, this is not renewed per attempt. */ - deadlineAt?: number; - /** Model/deployment scope; prevents one model cooldown from blocking another. */ - cooldownScope?: string; provider: EmbeddingProviderName; log: ProviderLogger; } @@ -39,33 +26,11 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< const maxRetries = opts.maxRetries ?? 2; let attempt = 0; let lastErr: unknown = null; - const cooldownKey = retryCooldownKey("embedding", opts.provider, opts.url, opts.cooldownScope); while (attempt <= maxRetries) { attempt++; const start = Date.now(); try { - const cooldown = getRetryCooldown(cooldownKey, start); - if (cooldown) { - const details = { - provider: opts.provider, - url: opts.url, - status: cooldown.status, - attempt, - maxRetries, - retryAfterMs: cooldown.retryAfterMs, - retryAt: cooldown.retryAt, - retryDecision: "defer", - retryReason: "cooldown_active", - remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt, start), - }; - opts.log.warn("http.retry_cooldown", details); - throw new MemosError( - ERROR_CODES.EMBEDDING_UNAVAILABLE, - `${opts.provider} is cooling down until ${new Date(cooldown.retryAt).toISOString()}`, - details, - ); - } const signal = mergeSignals(opts.signal, AbortSignal.timeout(timeoutMs)); const resp = await fetch(opts.url, { method: "POST", @@ -81,62 +46,21 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< if (!resp.ok) { const text = await safeText(resp); const transient = resp.status >= 500 || resp.status === 429; - const retryAfterMs = resp.status === 429 || resp.status === 503 - ? parseRetryAfterMs(resp.headers.get("Retry-After")) - : null; - if (retryAfterMs !== null) { - recordRetryCooldown(cooldownKey, { - retryAfterMs, - retryAt: Date.now() + retryAfterMs, - status: resp.status, - }); - } opts.log.warn("http.non_ok", { url: opts.url, status: resp.status, attempt, transient, - retryAfterMs, durationMs: Date.now() - start, }); if (transient && attempt <= maxRetries) { - const plan = planRetry({ - attempt, - baseMs: 200, - jitterMaxMs: 100, - retryAfterMs, - deadlineAt: opts.deadlineAt, - }); - const retryDetails = retryPlanDetails(plan, opts, maxRetries, resp.status, attempt); - if (plan.action === "defer") { - opts.log.warn("http.retry_deferred", retryDetails); - throw new MemosError( - ERROR_CODES.EMBEDDING_UNAVAILABLE, - `HTTP ${resp.status} from ${opts.provider}; retry deferred until ${new Date(plan.retryAt).toISOString()}`, - retryDetails, - ); - } - opts.log.warn("http.retry_scheduled", retryDetails); - await waitForRetry(plan.delayMs, opts.signal); + await backoff(attempt); continue; } throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, `HTTP ${resp.status} from ${opts.provider}`, - { - provider: opts.provider, - url: opts.url, - status: resp.status, - body: text, - ...(retryAfterMs === null - ? {} - : { - retryAfterMs, - retryAt: Date.now() + retryAfterMs, - retryDecision: "stop", - retryReason: "retries_exhausted", - }), - }, + { provider: opts.provider, url: opts.url, status: resp.status, body: text }, ); } @@ -159,23 +83,7 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< durationMs: Date.now() - start, }); if (transient && attempt <= maxRetries) { - const plan = planRetry({ - attempt, - baseMs: 200, - jitterMaxMs: 100, - deadlineAt: opts.deadlineAt, - }); - const retryDetails = retryPlanDetails(plan, opts, maxRetries, null, attempt); - if (plan.action === "defer") { - opts.log.warn("http.retry_deferred", retryDetails); - throw new MemosError( - ERROR_CODES.EMBEDDING_UNAVAILABLE, - `${opts.provider} retry cannot fit the request deadline`, - retryDetails, - ); - } - opts.log.warn("http.retry_scheduled", retryDetails); - await waitForRetry(plan.delayMs, opts.signal); + await backoff(attempt); continue; } throw new MemosError( @@ -215,32 +123,11 @@ function isTransientError(err: unknown): boolean { return false; } -function retryPlanDetails( - plan: RetryPlan, - opts: HttpPostOpts, - maxRetries: number, - status: number | null, - attempt: number, -): Record { - return { - provider: opts.provider, - url: opts.url, - status, - attempt, - maxRetries, - backoffMs: plan.backoffMs, - plannedDelayMs: plan.delayMs, - retryAfterMs: plan.retryAfterMs, - retryAt: plan.retryAt, - retrySource: plan.source, - retryDecision: plan.action, - ...(plan.action === "defer" ? { retryReason: plan.reason } : {}), - remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt), - }; -} - -function remainingDeadlineMs(deadlineAt?: number, nowMs: number = Date.now()): number | null { - return deadlineAt === undefined ? null : Math.max(0, deadlineAt - nowMs); +async function backoff(attempt: number): Promise { + const base = 200; + const jitter = Math.floor(Math.random() * 100); + const ms = base * 2 ** (attempt - 1) + jitter; + await new Promise((r) => setTimeout(r, ms)); } function mergeSignals(a: AbortSignal | undefined, b: AbortSignal): AbortSignal { diff --git a/apps/memos-local-plugin/core/embedding/index.ts b/apps/memos-local-plugin/core/embedding/index.ts index f6f4b1ed9..99faa0048 100644 --- a/apps/memos-local-plugin/core/embedding/index.ts +++ b/apps/memos-local-plugin/core/embedding/index.ts @@ -19,7 +19,6 @@ export { l2Normalize, enforceDim, postProcess, toFloat32 } from "./normalize.js" export { createEmbeddingRetryWorker, systemErrorEvent } from "./retry-worker.js"; export type { EmbeddingRetryWorker } from "./retry-worker.js"; export type { - EmbedCallOptions, EmbedInput, EmbedRole, EmbedStats, diff --git a/apps/memos-local-plugin/core/embedding/providers/cohere.ts b/apps/memos-local-plugin/core/embedding/providers/cohere.ts index 58214d341..891cd4506 100644 --- a/apps/memos-local-plugin/core/embedding/providers/cohere.ts +++ b/apps/memos-local-plugin/core/embedding/providers/cohere.ts @@ -21,7 +21,7 @@ export class CohereEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "cohere"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -49,8 +49,6 @@ export class CohereEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/gemini.ts b/apps/memos-local-plugin/core/embedding/providers/gemini.ts index 68ba22708..91d97acba 100644 --- a/apps/memos-local-plugin/core/embedding/providers/gemini.ts +++ b/apps/memos-local-plugin/core/embedding/providers/gemini.ts @@ -22,7 +22,7 @@ export class GeminiEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "gemini"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -50,8 +50,6 @@ export class GeminiEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/mistral.ts b/apps/memos-local-plugin/core/embedding/providers/mistral.ts index 7eace8ab7..21f50769f 100644 --- a/apps/memos-local-plugin/core/embedding/providers/mistral.ts +++ b/apps/memos-local-plugin/core/embedding/providers/mistral.ts @@ -23,7 +23,7 @@ export class MistralEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "mistral"; async embed(texts: string[], _role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -46,8 +46,6 @@ export class MistralEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/openai.ts b/apps/memos-local-plugin/core/embedding/providers/openai.ts index 57d852639..c0df47831 100644 --- a/apps/memos-local-plugin/core/embedding/providers/openai.ts +++ b/apps/memos-local-plugin/core/embedding/providers/openai.ts @@ -27,7 +27,7 @@ export class OpenAiEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "openai_compatible"; async embed(texts: string[], _role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -53,8 +53,6 @@ export class OpenAiEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/voyage.ts b/apps/memos-local-plugin/core/embedding/providers/voyage.ts index 6f36fe6d8..f89eca832 100644 --- a/apps/memos-local-plugin/core/embedding/providers/voyage.ts +++ b/apps/memos-local-plugin/core/embedding/providers/voyage.ts @@ -23,7 +23,7 @@ export class VoyageEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "voyage"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -50,8 +50,6 @@ export class VoyageEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/retry-worker.ts b/apps/memos-local-plugin/core/embedding/retry-worker.ts index 34db37bff..3620fd9e3 100644 --- a/apps/memos-local-plugin/core/embedding/retry-worker.ts +++ b/apps/memos-local-plugin/core/embedding/retry-worker.ts @@ -35,7 +35,6 @@ export function createEmbeddingRetryWorker( const workerId = `embedding-retry-${ids.span()}`; let timer: ReturnType | null = null; let running: Promise | null = null; - let stopped = false; async function runOnce(): Promise { if (!deps.embedder) return; @@ -104,11 +103,6 @@ export function createEmbeddingRetryWorker( const message = err instanceof Error ? err.message : String(err); const at = now(); const terminal = attemptNo >= job.maxAttempts; - const providerRetryAt = retryAtFromError(err, at); - const nextAttemptAt = Math.max( - at + backoffMs(attemptNo), - providerRetryAt ?? 0, - ); const recorded = terminal ? deps.repos.embeddingRetryQueue.markFailedClaimed(job.id, { ...claim, @@ -119,7 +113,7 @@ export function createEmbeddingRetryWorker( : deps.repos.embeddingRetryQueue.markRetryClaimed(job.id, { ...claim, attempts: attemptNo, - nextAttemptAt, + nextAttemptAt: at + backoffMs(attemptNo), error: message, now: at, }); @@ -127,10 +121,7 @@ export function createEmbeddingRetryWorker( deps.log.debug("embedding_retry.stale_failure_ignored", { jobId: job.id, terminal }); return; } - emitFailure(job, attemptNo, message, terminal, at, { - providerRetryAt, - nextAttemptAt: terminal ? null : nextAttemptAt, - }); + emitFailure(job, attemptNo, message, terminal, at); } } @@ -162,7 +153,6 @@ export function createEmbeddingRetryWorker( message: string, terminal: boolean, at: number, - retry: { providerRetryAt: number | null; nextAttemptAt: number | null }, ): void { const payload = { kind: "embedding.retry_failed", @@ -174,8 +164,6 @@ export function createEmbeddingRetryWorker( maxAttempts: job.maxAttempts, terminal, message, - providerRetryAt: retry.providerRetryAt, - nextAttemptAt: retry.nextAttemptAt, }; deps.log.warn("embedding_retry.failed", payload); try { @@ -194,7 +182,7 @@ export function createEmbeddingRetryWorker( } function tick(): void { - if (stopped || running) return; + if (running) return; running = runOnce().finally(() => { running = null; }); @@ -202,30 +190,21 @@ export function createEmbeddingRetryWorker( return { start(): void { - if (stopped || timer || !deps.embedder) return; + if (timer || !deps.embedder) return; tick(); timer = setInterval(tick, deps.intervalMs ?? DEFAULT_INTERVAL_MS); }, stop(): void { - stopped = true; if (timer) clearInterval(timer); timer = null; }, async flush(): Promise { - if (!stopped) tick(); + tick(); if (running) await running; }, }; } -function retryAtFromError(err: unknown, nowMs: number): number | null { - if (!err || typeof err !== "object") return null; - const details = (err as { details?: unknown }).details; - if (!details || typeof details !== "object") return null; - const retryAt = Number((details as { retryAt?: unknown }).retryAt); - return Number.isSafeInteger(retryAt) && retryAt > nowMs ? retryAt : null; -} - function backoffMs(attemptNo: number): number { return Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** Math.max(0, attemptNo - 1)); } diff --git a/apps/memos-local-plugin/core/embedding/types.ts b/apps/memos-local-plugin/core/embedding/types.ts index 95726703c..4f3f5eb99 100644 --- a/apps/memos-local-plugin/core/embedding/types.ts +++ b/apps/memos-local-plugin/core/embedding/types.ts @@ -6,7 +6,6 @@ */ import type { EmbeddingVector } from "../types.js"; -import type { RetryDiagnosticDetails } from "../util/retry-after.js"; // ─── Config ────────────────────────────────────────────────────────────────── @@ -63,7 +62,7 @@ export interface EmbeddingConfig { onStatus?: (detail: EmbeddingStatusDetail) => void; } -export interface EmbeddingErrorDetail extends RetryDiagnosticDetails { +export interface EmbeddingErrorDetail { kind: "embedding"; provider: EmbeddingProviderName | string; model: string; @@ -74,7 +73,7 @@ export interface EmbeddingErrorDetail extends RetryDiagnosticDetails { at?: number; } -export interface EmbeddingStatusDetail extends RetryDiagnosticDetails { +export interface EmbeddingStatusDetail { kind: "embedding"; status: "ok" | "error"; provider: EmbeddingProviderName | string; @@ -129,8 +128,6 @@ export interface ProviderCallCtx { log: ProviderLogger; /** AbortSignal honored across HTTP + native calls. */ signal?: AbortSignal; - /** Absolute end-to-end deadline shared across provider retry attempts. */ - deadlineAt?: number; } export interface ProviderLogger { @@ -169,16 +166,13 @@ export interface Embedder { /** Model identifier as configured by the operator (e.g. "bge-m3"). */ readonly model: string; - embedOne(input: string | EmbedInput, options?: EmbedCallOptions): Promise; + embedOne(input: string | EmbedInput): Promise; /** * Batch-embed many texts. Results keep input order. Duplicates are deduped * internally so a text repeated N times causes 1 cache miss max. */ - embedMany( - inputs: Array, - options?: EmbedCallOptions, - ): Promise; + embedMany(inputs: Array): Promise; stats(): EmbedStats; @@ -187,12 +181,6 @@ export interface Embedder { close(): Promise; } -export interface EmbedCallOptions { - signal?: AbortSignal; - /** Absolute end-to-end deadline shared across provider retry attempts. */ - deadlineAt?: number; -} - // ─── Errors ────────────────────────────────────────────────────────────────── export interface ProviderHttpFailure { diff --git a/apps/memos-local-plugin/core/index.ts b/apps/memos-local-plugin/core/index.ts index c5e4fab9d..d2b979bce 100644 --- a/apps/memos-local-plugin/core/index.ts +++ b/apps/memos-local-plugin/core/index.ts @@ -110,7 +110,6 @@ export { MistralEmbeddingProvider, type EmbedCache, type EmbedCacheStats, - type EmbedCallOptions, type EmbedInput, type EmbedRole, type EmbedStats, diff --git a/apps/memos-local-plugin/core/llm/client.ts b/apps/memos-local-plugin/core/llm/client.ts index ee456ac12..6bedafa70 100644 --- a/apps/memos-local-plugin/core/llm/client.ts +++ b/apps/memos-local-plugin/core/llm/client.ts @@ -18,7 +18,6 @@ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; import { rootLogger } from "../logger/index.js"; import type { Logger } from "../logger/types.js"; -import { extractRetryDiagnostics } from "../util/retry-after.js"; import { getHostLlmBridge } from "./host-bridge.js"; import { buildJsonSystemHint, parseLlmJson } from "./json-mode.js"; import { AnthropicLlmProvider } from "./providers/anthropic.js"; @@ -293,7 +292,6 @@ export function createLlmClientWithProvider( }, log: pLog, signal: opts?.signal, - deadlineAt: opts?.deadlineAt, }; } @@ -372,7 +370,6 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(hostErr), code: hostErr instanceof MemosError ? hostErr.code : undefined, - ...extractRetryDiagnostics(hostErr instanceof MemosError ? hostErr.details : undefined), at: failAt, durationMs: Date.now() - startedAt, fallbackProvider: "host", @@ -398,7 +395,6 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, - ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), at: failAt, durationMs: Date.now() - startedAt, op, @@ -429,7 +425,6 @@ export function createLlmClientWithProvider( message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, at: Date.now(), - ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); } catch { /* sink errors are non-fatal */ @@ -449,10 +444,6 @@ export function createLlmClientWithProvider( op?: string; episodeId?: string; phase?: string; - retryAfterMs?: number; - retryAt?: number; - retryDecision?: "wait" | "defer" | "stop"; - retryReason?: string; }): void { if (!config.onStatus) return; try { @@ -625,7 +616,6 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, - ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), at: failAt, durationMs: Date.now() - start, op: opts?.op ?? "stream", @@ -679,7 +669,6 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(primaryErr), code: primaryErr instanceof MemosError ? primaryErr.code : undefined, - ...extractRetryDiagnostics(primaryErr instanceof MemosError ? primaryErr.details : undefined), at: fallbackAt, durationMs: completion.durationMs, fallbackProvider: "host", diff --git a/apps/memos-local-plugin/core/llm/fetcher.ts b/apps/memos-local-plugin/core/llm/fetcher.ts index 358c73275..53eb55ec3 100644 --- a/apps/memos-local-plugin/core/llm/fetcher.ts +++ b/apps/memos-local-plugin/core/llm/fetcher.ts @@ -11,15 +11,6 @@ */ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; -import { - getRetryCooldown, - parseRetryAfterMs, - planRetry, - recordRetryCooldown, - retryCooldownKey, - type RetryPlan, - waitForRetry, -} from "../util/retry-after.js"; import type { LlmProviderLogger, LlmProviderName } from "./types.js"; export interface HttpPostOpts { @@ -29,10 +20,6 @@ export interface HttpPostOpts { timeoutMs: number; maxRetries: number; signal?: AbortSignal; - /** Absolute end-to-end deadline. Unlike timeoutMs, this is not renewed per attempt. */ - deadlineAt?: number; - /** Model/deployment scope; prevents one model cooldown from blocking another. */ - cooldownScope?: string; provider: LlmProviderName; log: LlmProviderLogger; onRetry?: (attempt: number) => void; @@ -48,33 +35,11 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< }> { let attempt = 0; let lastErr: unknown = null; - const cooldownKey = retryCooldownKey("llm", opts.provider, opts.url, opts.cooldownScope); while (attempt <= opts.maxRetries) { attempt++; const start = Date.now(); try { - const cooldown = getRetryCooldown(cooldownKey, start); - if (cooldown) { - const details = { - provider: opts.provider, - url: opts.url, - status: cooldown.status, - attempt, - maxRetries: opts.maxRetries, - retryAfterMs: cooldown.retryAfterMs, - retryAt: cooldown.retryAt, - retryDecision: "defer", - retryReason: "cooldown_active", - remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt, start), - }; - opts.log.warn("http.retry_cooldown", details); - throw new MemosError( - errCodeForStatus(cooldown.status), - `${opts.provider} is cooling down until ${new Date(cooldown.retryAt).toISOString()}`, - details, - ); - } const signal = mergeSignals(opts.signal, AbortSignal.timeout(opts.timeoutMs)); const resp = await fetch(opts.url, { method: "POST", @@ -91,63 +56,22 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< if (!resp.ok) { const text = await safeText(resp); const transient = resp.status >= 500 || resp.status === 429; - const retryAfterMs = resp.status === 429 || resp.status === 503 - ? parseRetryAfterMs(resp.headers.get("Retry-After")) - : null; - if (retryAfterMs !== null) { - recordRetryCooldown(cooldownKey, { - retryAfterMs, - retryAt: Date.now() + retryAfterMs, - status: resp.status, - }); - } opts.log.warn("http.non_ok", { status: resp.status, attempt, transient, durationMs: ms, - retryAfterMs, body: truncateLogBody(text), }); if (transient && attempt <= opts.maxRetries) { - const plan = planRetry({ - attempt, - baseMs: 250, - jitterMaxMs: 120, - retryAfterMs, - deadlineAt: opts.deadlineAt, - }); - const retryDetails = retryPlanDetails(plan, opts, resp.status, attempt); - if (plan.action === "defer") { - opts.log.warn("http.retry_deferred", retryDetails); - throw new MemosError( - errCodeForStatus(resp.status), - `HTTP ${resp.status} from ${opts.provider}; retry deferred until ${new Date(plan.retryAt).toISOString()}`, - retryDetails, - ); - } - opts.log.warn("http.retry_scheduled", retryDetails); opts.onRetry?.(attempt); - await waitForRetry(plan.delayMs, opts.signal); + await backoff(attempt); continue; } throw new MemosError( errCodeForStatus(resp.status), `HTTP ${resp.status} from ${opts.provider}`, - { - provider: opts.provider, - url: opts.url, - status: resp.status, - body: text, - ...(retryAfterMs === null - ? {} - : { - retryAfterMs, - retryAt: Date.now() + retryAfterMs, - retryDecision: "stop", - retryReason: "retries_exhausted", - }), - }, + { provider: opts.provider, url: opts.url, status: resp.status, body: text }, ); } @@ -161,15 +85,8 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< } catch (err) { lastErr = err; if (err instanceof MemosError) throw err; - if (opts.signal?.aborted) { - throw new MemosError( - ERROR_CODES.LLM_TIMEOUT, - `${opts.provider} request was cancelled`, - { provider: opts.provider, url: opts.url, cancelled: true }, - ); - } const transient = isTransientError(err); - const timedOut = isTimeout(err) || opts.signal?.aborted === true; + const timedOut = isTimeout(err); opts.log.warn("http.exception", { attempt, transient, @@ -177,24 +94,8 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< err: toErrDetail(err), }); if ((transient || timedOut) && attempt <= opts.maxRetries) { - const plan = planRetry({ - attempt, - baseMs: 250, - jitterMaxMs: 120, - deadlineAt: opts.deadlineAt, - }); - const retryDetails = retryPlanDetails(plan, opts, null, attempt); - if (plan.action === "defer") { - opts.log.warn("http.retry_deferred", retryDetails); - throw new MemosError( - timedOut ? ERROR_CODES.LLM_TIMEOUT : ERROR_CODES.LLM_UNAVAILABLE, - `${opts.provider} retry cannot fit the request deadline`, - retryDetails, - ); - } - opts.log.warn("http.retry_scheduled", retryDetails); opts.onRetry?.(attempt); - await waitForRetry(plan.delayMs, opts.signal); + await backoff(attempt); continue; } if (timedOut) { @@ -344,31 +245,11 @@ function isTimeout(err: unknown): boolean { return false; } -function retryPlanDetails( - plan: RetryPlan, - opts: HttpPostOpts, - status: number | null, - attempt: number, -): Record { - return { - provider: opts.provider, - url: opts.url, - status, - attempt, - maxRetries: opts.maxRetries, - backoffMs: plan.backoffMs, - plannedDelayMs: plan.delayMs, - retryAfterMs: plan.retryAfterMs, - retryAt: plan.retryAt, - retrySource: plan.source, - retryDecision: plan.action, - ...(plan.action === "defer" ? { retryReason: plan.reason } : {}), - remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt), - }; -} - -function remainingDeadlineMs(deadlineAt?: number, nowMs: number = Date.now()): number | null { - return deadlineAt === undefined ? null : Math.max(0, deadlineAt - nowMs); +async function backoff(attempt: number): Promise { + const base = 250; + const jitter = Math.floor(Math.random() * 120); + const ms = base * 2 ** (attempt - 1) + jitter; + await new Promise((r) => setTimeout(r, ms)); } function mergeSignals(a: AbortSignal | undefined, b: AbortSignal): AbortSignal { diff --git a/apps/memos-local-plugin/core/llm/providers/anthropic.ts b/apps/memos-local-plugin/core/llm/providers/anthropic.ts index 6c510ef75..66a9ee446 100644 --- a/apps/memos-local-plugin/core/llm/providers/anthropic.ts +++ b/apps/memos-local-plugin/core/llm/providers/anthropic.ts @@ -31,7 +31,7 @@ export class AnthropicLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -68,8 +68,6 @@ export class AnthropicLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/bedrock.ts b/apps/memos-local-plugin/core/llm/providers/bedrock.ts index d956b7111..7c00470e7 100644 --- a/apps/memos-local-plugin/core/llm/providers/bedrock.ts +++ b/apps/memos-local-plugin/core/llm/providers/bedrock.ts @@ -36,7 +36,7 @@ export class BedrockLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.endpoint || config.endpoint.length === 0) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -85,8 +85,6 @@ export class BedrockLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/gemini.ts b/apps/memos-local-plugin/core/llm/providers/gemini.ts index 6bfa323eb..4e6273b55 100644 --- a/apps/memos-local-plugin/core/llm/providers/gemini.ts +++ b/apps/memos-local-plugin/core/llm/providers/gemini.ts @@ -39,7 +39,7 @@ export class GeminiLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -59,8 +59,6 @@ export class GeminiLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/openai.ts b/apps/memos-local-plugin/core/llm/providers/openai.ts index 562521756..d8a5a20af 100644 --- a/apps/memos-local-plugin/core/llm/providers/openai.ts +++ b/apps/memos-local-plugin/core/llm/providers/openai.ts @@ -51,7 +51,7 @@ export class OpenAiLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal, deadlineAt } = ctx; + const { config, log, signal } = ctx; const url = normalizeEndpoint( config.endpoint && config.endpoint.length > 0 ? config.endpoint @@ -93,8 +93,6 @@ export class OpenAiLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, - deadlineAt, - cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/types.ts b/apps/memos-local-plugin/core/llm/types.ts index 2dd761f32..a37a2677d 100644 --- a/apps/memos-local-plugin/core/llm/types.ts +++ b/apps/memos-local-plugin/core/llm/types.ts @@ -6,7 +6,6 @@ */ import type { ReasoningConfig as ConfigReasoningConfig } from "../config/schema.js"; -import type { RetryDiagnosticDetails } from "../util/retry-after.js"; // ─── Providers & config ────────────────────────────────────────────────────── @@ -89,7 +88,7 @@ export interface LlmCircuitBreakerConfig { now?: () => number; } -export interface LlmErrorDetail extends RetryDiagnosticDetails { +export interface LlmErrorDetail { provider: LlmProviderName | string; model: string; message: string; @@ -106,7 +105,7 @@ export interface LlmErrorDetail extends RetryDiagnosticDetails { role?: "llm" | "skillEvolver"; } -export interface LlmStatusDetail extends RetryDiagnosticDetails { +export interface LlmStatusDetail { status: "ok" | "fallback" | "error" | "circuit_open"; provider: LlmProviderName | string; model: string; @@ -150,8 +149,6 @@ export interface LlmCallOptions { maxTokens?: number; /** Per-call timeout. */ timeoutMs?: number; - /** Absolute end-to-end deadline shared across provider retry attempts. */ - deadlineAt?: number; /** AbortSignal honored across HTTP + host-bridge calls. */ signal?: AbortSignal; /** @@ -216,8 +213,6 @@ export interface LlmProviderCtx { log: LlmProviderLogger; /** Call abort signal; providers must honor it. */ signal?: AbortSignal; - /** Absolute end-to-end deadline; providers must not renew it per retry. */ - deadlineAt?: number; } export interface LlmProviderLogger { diff --git a/apps/memos-local-plugin/core/pipeline/deps.ts b/apps/memos-local-plugin/core/pipeline/deps.ts index 79714b35e..8a0772119 100644 --- a/apps/memos-local-plugin/core/pipeline/deps.ts +++ b/apps/memos-local-plugin/core/pipeline/deps.ts @@ -101,10 +101,6 @@ import type { import { wrapRetrievalRepos } from "./retrieval-repos.js"; import { createSemaphore } from "../util/semaphore.js"; import { rateLimitLlmClient } from "../util/rate-limited-llm.js"; -import { - prioritizeEmbedder, - type ForegroundResources, -} from "../util/foreground-resources.js"; // ─── Algorithm config slice helper ──────────────────────────────────────── @@ -212,23 +208,19 @@ export function buildPipelineSubscribers( buses: PipelineBuses, algorithm: PipelineAlgorithmConfig, session?: PipelineSessionSet, - resources?: ForegroundResources, ): PipelineSubscriberSet { const log = deps.log ?? rootLogger.child({ channel: "core.pipeline" }); const bgLlmSemaphore = createSemaphore(algorithm.session.bgLlmConcurrency); - const bgLlm = rateLimitLlmClient(deps.llm, bgLlmSemaphore, resources); - const bgReflectLlm = rateLimitLlmClient(deps.reflectLlm, bgLlmSemaphore, resources); - const bgL3Llm = rateLimitLlmClient(deps.l3Llm ?? deps.llm, bgLlmSemaphore, resources); - const bgEmbedder = resources - ? prioritizeEmbedder(deps.embedder, resources, "background") - : deps.embedder; + const bgLlm = rateLimitLlmClient(deps.llm, bgLlmSemaphore); + const bgReflectLlm = rateLimitLlmClient(deps.reflectLlm, bgLlmSemaphore); + const bgL3Llm = rateLimitLlmClient(deps.l3Llm ?? deps.llm, bgLlmSemaphore); const lightweightMode = algorithm.lightweightMemory.enabled; const captureRunner = createCaptureRunner({ tracesRepo: deps.repos.traces, embeddingRetryQueue: deps.repos.embeddingRetryQueue, episodesRepo: adaptEpisodesRepo(deps.repos.episodes), - embedder: bgEmbedder, + embedder: deps.embedder, llm: bgLlm, // Issue #2148: capture batch reflection emits JSON, so it must use // the main model rather than the potentially thinking-enabled @@ -335,7 +327,7 @@ export function buildPipelineSubscribers( const skillHandle = attachSkillSubscriber({ repos: deps.repos, - embedder: bgEmbedder, + embedder: deps.embedder, llm: bgLlm, bus: buses.skill, l2Bus: buses.l2, @@ -347,7 +339,7 @@ export function buildPipelineSubscribers( const feedbackHandle = attachFeedbackSubscriber({ repos: deps.repos, llm: bgLlm, - embedder: bgEmbedder, + embedder: deps.embedder, bus: buses.feedback, log: log.child({ channel: "core.feedback" }), config: algorithm.feedback, @@ -412,17 +404,14 @@ export function buildPipelineSession( export function buildRetrievalDeps( deps: PipelineDeps, algorithm: PipelineAlgorithmConfig, - resources?: ForegroundResources, ): RetrievalDeps { - const embedder = resources - ? prioritizeEmbedder(deps.embedder, resources, "foreground") - : deps.embedder; + const embedder = deps.embedder; return { repos: wrapRetrievalRepos(deps.repos, deps.namespace), embedder: embedder ? { - embed: (text, role, options) => - embedder.embedOne({ text, role: role ?? "query" }, options), + embed: (text, role) => + embedder.embedOne({ text, role: role ?? "query" }), } : { // Degraded mode: empty vector so vector-scoring falls back to diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index a0354bf64..c9254e092 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -1232,59 +1232,18 @@ export function createMemoryCore( const statsLine = `phase=${phase}, stored=${storedCount}` + (r.warnings.length > 0 ? `, warnings=${r.warnings.length}` : ""); - const action = phase === "lite" - ? ("stored" as const) - : ("reflected" as const); - const details = r.traces.flatMap((tc) => { - const items: Array<{ - role: "user" | "assistant" | "tool" | "reflection" | "other"; - action: typeof action; - summary: string | null; - content: string; - traceId: string; - }> = []; - - if (tc.userText) { - items.push({ - role: "user", - action, - summary: null, - content: tc.userText.slice(0, 400), - traceId: tc.traceId, - }); - } - if (tc.agentText) { - items.push({ - role: "assistant", - action, - summary: null, - content: tc.agentText.slice(0, 400), - traceId: tc.traceId, - }); - } - - const toolSummary = summarizeToolCalls(tc.toolCalls); - if (items.length === 0) { - items.push({ - role: toolSummary ? "tool" : "other", - action, - summary: tc.reflection?.text ?? null, - content: toolSummary.slice(0, 400), - traceId: tc.traceId, - }); - } else if (tc.reflection?.text) { - // Keep the existing reflect-phase summary visible without - // presenting it as either side's original chat content. - items.push({ - role: "reflection", - action, - summary: tc.reflection.text, - content: "", - traceId: tc.traceId, - }); - } - return items; - }); + const details = r.traces.map((tc) => ({ + role: inferTurnRole(tc), + action: phase === "lite" ? ("stored" as const) : ("reflected" as const), + summary: tc.reflection?.text ?? null, + content: ( + tc.userText || + tc.agentText || + summarizeToolCalls(tc.toolCalls) || + "" + ).slice(0, 400), + traceId: tc.traceId, + })); handle.repos.apiLogs.insert({ toolName: "memory_add", input: { @@ -6236,3 +6195,26 @@ function summarizeToolCalls( }) .join("\n"); } + +/** + * Heuristic role inference for api_logs "memory_add" rows — mirrors + * the legacy plugin's behaviour where each captured turn showed up + * labelled `user` / `assistant` / `tool` on the Logs page. + * + * Priority: if the step carries userText (the user's query), label it + * "user" even when toolCalls are present — this is the first sub-step + * of a multi-tool turn and semantically represents the user request. + */ +function inferTurnRole(step: { + userText?: string; + agentText?: string; + toolCalls?: readonly unknown[]; +}): "user" | "assistant" | "tool" | "other" { + const u = (step.userText ?? "").length; + const a = (step.agentText ?? "").length; + if (u > 0 && (step.toolCalls?.length ?? 0) > 0) return "user"; + if ((step.toolCalls?.length ?? 0) > 0) return "tool"; + if (u >= a && u > 0) return "user"; + if (a > 0) return "assistant"; + return "other"; +} diff --git a/apps/memos-local-plugin/core/pipeline/orchestrator.ts b/apps/memos-local-plugin/core/pipeline/orchestrator.ts index 4ed0427d1..ac48133ad 100644 --- a/apps/memos-local-plugin/core/pipeline/orchestrator.ts +++ b/apps/memos-local-plugin/core/pipeline/orchestrator.ts @@ -83,39 +83,29 @@ import { onBroadcastLog } from "../logger/transports/sse-broadcast.js"; import { createEmbeddingRetryWorker, systemErrorEvent } from "../embedding/index.js"; import type { EpisodeSnapshot } from "../session/index.js"; import type { IntentDecision, RelationDecision, TurnRelation } from "../session/types.js"; -import { - createForegroundResources, - prioritizeEmbedder, -} from "../util/foreground-resources.js"; -import { createRequestDeadline } from "../util/request-deadline.js"; function classifyWithTimeout( classifyFn: () => Promise, timeoutMs: number, log: Logger, ): Promise { - let timer: ReturnType | null = null; return Promise.race([ classifyFn(), - new Promise((_, reject) => { - timer = setTimeout(() => reject(new Error("classify_timeout")), timeoutMs); - }), - ]) - .catch((err) => { - log.warn("relation.classify_timeout", { - timeoutMs, - err: err instanceof Error ? err.message : String(err), - }); - return { - relation: "follow_up" as const, - confidence: 0, - reason: "classify_timeout", - signals: ["classify_timeout"], - }; - }) - .finally(() => { - if (timer) clearTimeout(timer); + new Promise((_, reject) => + setTimeout(() => reject(new Error("classify_timeout")), timeoutMs), + ), + ]).catch((err) => { + log.warn("relation.classify_timeout", { + timeoutMs, + err: err instanceof Error ? err.message : String(err), }); + return { + relation: "new_task" as const, + confidence: 0, + reason: "classify_timeout", + signals: ["classify_timeout"], + }; + }); } // ─── Factory ────────────────────────────────────────────────────────────── @@ -125,12 +115,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { const algorithm = extractAlgorithmConfig(deps); const lightweightMode = algorithm.lightweightMemory.enabled; const buses = buildPipelineBuses(); - const foregroundResources = createForegroundResources(); - const backgroundEmbedder = prioritizeEmbedder( - deps.embedder, - foregroundResources, - "background", - ); // Session + intent. const session = buildPipelineSession(deps, buses.session); @@ -139,13 +123,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { // Pass `session` so the reward runner's `getEpisodeSnapshot` hook // can resolve the live, in-memory episode (with turns populated) // rather than falling back to the empty row from SQLite. - const subs = buildPipelineSubscribers( - deps, - buses, - algorithm, - session, - foregroundResources, - ); + const subs = buildPipelineSubscribers(deps, buses, algorithm, session); // Core-event aggregator. Every internal bus funnels into one stream. const eventListeners = new Set<(e: CoreEvent) => void>(); @@ -182,7 +160,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { let retryEventSeq = 1_000_000; const embeddingRetryWorker = createEmbeddingRetryWorker({ repos: deps.repos, - embedder: backgroundEmbedder, + embedder: deps.embedder, log: log.child({ channel: "core.embedding.retry" }), now: deps.now, onSystemError: (payload, correlationId) => { @@ -388,7 +366,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userText: string, meta: Record, turnTs?: number, - signal?: AbortSignal, ): Promise { const currentEpId = openEpisodeBySession.get(sessionId); if (currentEpId) { @@ -410,7 +387,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: lightweightEpisodeMeta(meta), - signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return snap; @@ -457,20 +433,13 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userText: string, meta: Record, agent: AgentKind, - signal?: AbortSignal, ): Promise<{ episode: EpisodeSnapshot; sessionId: SessionId; relation?: string }> { const mergeMode = algorithm.session.followUpMode === "merge_follow_ups"; const mergeCapMs = algorithm.session.mergeMaxGapMs; const turnTs = timestampFromMeta(meta, "startedAtTurnTs"); if (lightweightMode) { - const snap = await startLightweightEpisode( - sessionId, - userText, - meta, - turnTs, - signal, - ); + const snap = await startLightweightEpisode(sessionId, userText, meta, turnTs); return { episode: snap, sessionId, relation: "lightweight_memory" }; } @@ -496,7 +465,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: currentEpId, - signal, }), algorithm.session.classifyTimeoutMs, log, @@ -614,7 +582,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: "new_task" }, - signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -634,7 +601,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: decision.relation, gapMs }, - signal, }); openEpisodeBySession.set(sessionId, fresh.id as EpisodeId); return { episode: fresh, sessionId, relation: decision.relation }; @@ -679,7 +645,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: snapshot.id as EpisodeId, - signal, }), algorithm.session.classifyTimeoutMs, log, @@ -783,7 +748,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta, - signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: "bootstrap" }; @@ -798,7 +762,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: prev.episodeId, - signal, }), algorithm.session.classifyTimeoutMs, log, @@ -898,7 +861,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: "new_task" }, - signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -909,7 +871,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: decision.relation }, - signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -1082,7 +1043,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { // ─── Retrieval entry points ───────────────────────────────────────────── - const retrievalDeps = buildRetrievalDeps(deps, algorithm, foregroundResources); + const retrievalDeps = buildRetrievalDeps(deps, algorithm); const turnStartRetrievalStats = new Map(); function retrievalDepsFor(namespace = deps.namespace): typeof retrievalDeps { @@ -1096,7 +1057,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { async function retrieveTurnStart( input: TurnInputDTO, plan?: RetrievePlan, - signal?: AbortSignal, ): Promise { const ctx = { reason: "turn_start" as const, @@ -1113,8 +1073,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { { events: buses.retrieval, skipLlmFilter: input.contextHints?.__memosDeferLlmFilterToCaller === true, - signal, - deadlineAt: input.deadlineAt, plan: plan ? { scenarioId: plan.scenarioId, @@ -1217,45 +1175,13 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { } async function onTurnStartOnce(input: TurnInputDTO): Promise { - const leaveForeground = foregroundResources.enterForeground(); - const deadline = - input.deadlineAt === undefined - ? null - : createRequestDeadline(input.deadlineAt); - const startedAt = Date.now(); - let stage = "ensure_session"; - try { - return await onTurnStartForeground(input, deadline?.signal, (next) => { - stage = next; - }); - } finally { - if (deadline?.signal.aborted) { - log.warn("turn.start.deadline_exceeded", { - sessionId: input.sessionId, - deadlineAt: input.deadlineAt, - elapsedMs: Date.now() - startedAt, - stage, - }); - } - deadline?.dispose(); - leaveForeground(); - } - } - - async function onTurnStartForeground( - input: TurnInputDTO, - signal?: AbortSignal, - setStage: (stage: string) => void = () => {}, - ): Promise { const t0 = now(); - setStage("ensure_session"); const initialSessionId = await ensureSession( input.agent, input.sessionId, input.contextHints, ); - setStage("relation_and_episode"); const routing = await openEpisodeIfNeeded( initialSessionId, input.userText, @@ -1266,7 +1192,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { startedAtTurnTs: input.ts, }, input.agent, - signal, ); const sessionId = routing.sessionId; @@ -1278,12 +1203,10 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { sessionId, episodeId: episode.id as EpisodeId, }; - setStage("intent"); const schedulerIntent = await intentForCurrentTurn({ episode, userText: input.userText, ts: input.ts, - signal, }); const retrievePlan = scheduleInjection({ userText: input.userText, @@ -1317,13 +1240,10 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { retrievalTotalMs: 0, elapsedMs: now() - t0, }); - setStage("complete"); return packet; } - setStage("retrieval"); - const packet = await retrieveTurnStart(normalized, retrievePlan, signal); - setStage("complete"); + const packet = await retrieveTurnStart(normalized, retrievePlan); // Always stamp the routed sessionId + episodeId on the packet so // adapters can correlate the subsequent `agent_end` / `turn.end` // call without needing a separate round-trip to the session @@ -1582,28 +1502,12 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { async function shutdown(reason: string = "shutdown"): Promise { log.info("pipeline.shutdown.begin", { reason }); - // Stop admitting retry jobs, but preserve a bounded grace period for raw - // capture and downstream enrichment. Hermes' bridge owns a 20s outer - // shutdown ceiling, so abort before that rather than either hanging or - // discarding every single-shot session's enrichment immediately. - embeddingRetryWorker.stop(); - const flushPromise = flush(); try { - const completed = await settlesWithin(flushPromise, 15_000); - if (!completed) { - log.warn("pipeline.flush_timeout", { reason, timeoutMs: 15_000 }); - foregroundResources.shutdown(reason); - const aborted = await settlesWithin(flushPromise, 4_000); - if (!aborted) { - log.warn("pipeline.flush_abandoned", { reason, abortWaitMs: 4_000 }); - } - } + await flush(); } catch (err) { log.warn("pipeline.flush_failed", { err: err instanceof Error ? err.message : String(err), }); - } finally { - foregroundResources.shutdown(reason); } // Detach subscribers — prevents late events from re-queuing work. subs.subscriptions.capture.stop(); @@ -1612,26 +1516,13 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { subs.l3.detach(); subs.skills.dispose(); subs.feedback.dispose(); + embeddingRetryWorker.stop(); bridge.dispose(); logSubscription(); session.sessionManager.shutdown(reason); log.info("pipeline.shutdown.done", { reason }); } - async function settlesWithin(promise: Promise, timeoutMs: number): Promise { - let timer: ReturnType | null = null; - try { - return await Promise.race([ - promise.then(() => true), - new Promise((resolve) => { - timer = setTimeout(() => resolve(false), timeoutMs); - }), - ]); - } finally { - if (timer) clearTimeout(timer); - } - } - function now(): number { return (deps.now ?? Date.now)(); } @@ -1698,7 +1589,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { episode: EpisodeSnapshot; userText: string; ts?: number; - signal?: AbortSignal; }): Promise { const firstTurn = input.episode.turns[0]; const isFreshEpisodeForThisTurn = @@ -1713,7 +1603,6 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { return session.intent.classify(input.userText, { episodeId: input.episode.id as EpisodeId, - signal: input.signal, }); } diff --git a/apps/memos-local-plugin/core/retrieval/llm-filter.ts b/apps/memos-local-plugin/core/retrieval/llm-filter.ts index f665cb954..142bd626d 100644 --- a/apps/memos-local-plugin/core/retrieval/llm-filter.ts +++ b/apps/memos-local-plugin/core/retrieval/llm-filter.ts @@ -50,8 +50,6 @@ export interface FilterDeps { llm: LlmClient | null; log: Logger; timeoutMs?: number; - deadlineAt?: number; - signal?: AbortSignal; config: Pick< RetrievalConfig, | "llmFilterEnabled" @@ -119,10 +117,6 @@ export async function llmFilterCandidates( if (!deps.llm) { return passthrough(ranked, "no_llm"); } - if (deps.signal?.aborted) { - deps.log.debug("llm_filter.deadline_exceeded", { candidateCount: ranked.length }); - return safeCutoff(ranked, deps); - } const bodyChars = deps.config.llmFilterCandidateBodyChars ?? DEFAULT_CANDIDATE_BODY_CHARS; @@ -154,8 +148,6 @@ ${list}`, episodeId: input.episodeId, temperature: 0, timeoutMs: deps.timeoutMs, - deadlineAt: deps.deadlineAt, - signal: deps.signal, // Output is only ordered indices + one bool, but the list can // legitimately be as long as the ranked candidates. maxTokens: filterOutputTokenBudget(ranked.length), diff --git a/apps/memos-local-plugin/core/retrieval/retrieve.ts b/apps/memos-local-plugin/core/retrieval/retrieve.ts index ff95debf4..cec076f62 100644 --- a/apps/memos-local-plugin/core/retrieval/retrieve.ts +++ b/apps/memos-local-plugin/core/retrieval/retrieve.ts @@ -75,10 +75,6 @@ export interface RetrieveOptions { * one unified final LLM filter across all routes. */ skipLlmFilter?: boolean; - /** Shared foreground cancellation signal. */ - signal?: AbortSignal; - /** Absolute request deadline used to cap optional LLM filtering. */ - deadlineAt?: number; } export interface RetrievePlanOverride { @@ -262,28 +258,22 @@ async function runAll( degraded: false, }; const queryVec = compiled.text - ? await deps.embedder - .embed(compiled.text, "query", { - signal: opts.signal, - deadlineAt: opts.deadlineAt, - }) - .then((vec) => { - embeddingStats.ok = true; - return vec; - }) - .catch((err) => { - const code = (err as { code?: string })?.code; - const message = err instanceof Error ? err.message : String(err); - embeddingStats.degraded = true; - embeddingStats.errorCode = code; - embeddingStats.errorMessage = message; - log.warn("embed_failed", { - reason: ctx.reason, - code, - err: message, - }); - return null; - }) + ? await deps.embedder.embed(compiled.text, "query").then((vec) => { + embeddingStats.ok = true; + return vec; + }).catch((err) => { + const code = (err as { code?: string })?.code; + const message = err instanceof Error ? err.message : String(err); + embeddingStats.degraded = true; + embeddingStats.errorCode = code; + embeddingStats.errorMessage = message; + log.warn("embed_failed", { + reason: ctx.reason, + code, + err: message, + }); + return null; + }) : null; // The keyword channels (FTS + pattern) work even without an embedder, @@ -425,9 +415,6 @@ async function runAll( llm: deps.llm ?? null, log, config: deps.config, - signal: opts.signal, - deadlineAt: opts.deadlineAt, - timeoutMs: filterTimeoutMs(opts.deadlineAt), }, ); @@ -485,9 +472,6 @@ async function runAll( llm: deps.llm ?? null, log, config: deps.config, - signal: opts.signal, - deadlineAt: opts.deadlineAt, - timeoutMs: filterTimeoutMs(opts.deadlineAt), }, ); @@ -684,11 +668,6 @@ async function runAll( } } -function filterTimeoutMs(deadlineAt?: number): number | undefined { - if (deadlineAt === undefined) return undefined; - return Math.max(1, Math.min(2_000, deadlineAt - Date.now())); -} - function emptyResult( reason: RetrievalReason, agent: AgentKind, diff --git a/apps/memos-local-plugin/core/retrieval/types.ts b/apps/memos-local-plugin/core/retrieval/types.ts index fe700c611..8ed24935e 100644 --- a/apps/memos-local-plugin/core/retrieval/types.ts +++ b/apps/memos-local-plugin/core/retrieval/types.ts @@ -679,11 +679,7 @@ export interface RetrievalRepos { /** Abstract embedder surface consumed by retrieval. Mirrors `Embedder`. */ export interface RetrievalEmbedder { - embed: ( - text: string, - role?: "query" | "document", - options?: { signal?: AbortSignal; deadlineAt?: number }, - ) => Promise; + embed: (text: string, role?: "query" | "document") => Promise; } export interface RetrievalDeps { diff --git a/apps/memos-local-plugin/core/session/intent-classifier.ts b/apps/memos-local-plugin/core/session/intent-classifier.ts index a4896e07b..41206184d 100644 --- a/apps/memos-local-plugin/core/session/intent-classifier.ts +++ b/apps/memos-local-plugin/core/session/intent-classifier.ts @@ -46,8 +46,6 @@ export interface IntentClassifierOptions { export interface IntentClassifyOptions { /** Episode id this classification is being run for, when known. */ episodeId?: EpisodeId; - /** Foreground request cancellation propagated to the provider call. */ - signal?: AbortSignal; } export interface IntentClassifier { @@ -93,7 +91,7 @@ export function createIntentClassifier(opts: IntentClassifierOptions = {}): Inte if (!llmDisabled && llm) { try { const result = await withTimeout( - callLlm(llm, text, options?.episodeId, timeoutMs, options?.signal), + callLlm(llm, text, options?.episodeId), timeoutMs, "intent.llm.timeout", ); @@ -209,8 +207,6 @@ async function callLlm( llm: LlmClient, text: string, episodeId?: EpisodeId, - timeoutMs?: number, - signal?: AbortSignal, ): Promise { const rsp = await llm.completeJson<{ kind: unknown; confidence: unknown; reason: unknown }>( [ @@ -221,8 +217,6 @@ async function callLlm( op: "session.intent.classify", phase: "session", episodeId, - timeoutMs, - signal, schemaHint: `{"kind":"task"|"memory_probe"|"chitchat"|"meta"|"unknown","confidence":0..1,"reason":"..."}`, validate: (v) => { const o = v as Record; diff --git a/apps/memos-local-plugin/core/session/manager.ts b/apps/memos-local-plugin/core/session/manager.ts index 8f0f2e79f..44da570b2 100644 --- a/apps/memos-local-plugin/core/session/manager.ts +++ b/apps/memos-local-plugin/core/session/manager.ts @@ -60,8 +60,6 @@ export interface StartEpisodeInput { /** Adapter-provided event time for the first user turn. */ ts?: EpochMs; meta?: Record; - /** Foreground cancellation propagated to intent classification. */ - signal?: AbortSignal; } export interface SessionManager { @@ -269,7 +267,6 @@ export function createSessionManager(deps: SessionManagerDeps): SessionManager { const episodeId = (input.id ?? ids.episode()) as EpisodeId; const intent = await deps.intentClassifier.classify(input.userMessage, { episodeId, - signal: input.signal, }); // Wrap the write+emit in a log context so downstream listeners inherit diff --git a/apps/memos-local-plugin/core/session/relation-classifier.ts b/apps/memos-local-plugin/core/session/relation-classifier.ts index f0e1e2a24..bac4679e0 100644 --- a/apps/memos-local-plugin/core/session/relation-classifier.ts +++ b/apps/memos-local-plugin/core/session/relation-classifier.ts @@ -306,11 +306,7 @@ export function createRelationClassifier( // Step 2: LLM classification. if (!llmDisabled && opts.llm) { try { - const result = await withTimeout( - callLlm(opts.llm, input, timeoutMs), - timeoutMs, - "relation.llm.timeout", - ); + const result = await withTimeout(callLlm(opts.llm, input), timeoutMs, "relation.llm.timeout"); log.debug("llm.ok", { relation: result.relation, confidence: result.confidence, @@ -330,7 +326,7 @@ export function createRelationClassifier( }); try { const arb = await withTimeout( - callArbitration(opts.llm, input, timeoutMs), + callArbitration(opts.llm, input), timeoutMs, "relation.arbitration.timeout", ); @@ -520,11 +516,7 @@ function buildLlmUserContent(input: RelationInput): string { return parts.join("\n\n"); } -async function callLlm( - llm: LlmClient, - input: RelationInput, - timeoutMs?: number, -): Promise { +async function callLlm(llm: LlmClient, input: RelationInput): Promise { const userContent = buildLlmUserContent(input); const rsp = await llm.completeJson<{ relation: unknown; confidence: unknown; reason: unknown }>( @@ -536,8 +528,6 @@ async function callLlm( op: "session.relation.classify", phase: "session", episodeId: input.prevEpisodeId, - timeoutMs, - signal: input.signal, schemaHint: `{"relation":"revision"|"follow_up"|"new_task"|"unknown","confidence":0..1,"reason":"..."}`, validate: (v) => { const o = v as Record; @@ -591,11 +581,7 @@ When in doubt, choose follow_up. Reply JSON ONLY: {"relation":"follow_up"|"new_task","reason":"..."}`; -async function callArbitration( - llm: LlmClient, - input: RelationInput, - timeoutMs?: number, -): Promise { +async function callArbitration(llm: LlmClient, input: RelationInput): Promise { const userContent = [ `CURRENT TASK CONTEXT:\n${(input.prevUserText ?? "").slice(0, 600)}`, `ASSISTANT REPLY:\n${(input.prevAssistantText ?? "").slice(0, 800)}`, @@ -611,8 +597,6 @@ async function callArbitration( op: "session.relation.arbitrate", phase: "session", episodeId: input.prevEpisodeId, - timeoutMs, - signal: input.signal, schemaHint: `{"relation":"follow_up"|"new_task","reason":"..."}`, validate: (v) => { const o = v as Record; diff --git a/apps/memos-local-plugin/core/session/types.ts b/apps/memos-local-plugin/core/session/types.ts index 62ea53f97..3d34438ea 100644 --- a/apps/memos-local-plugin/core/session/types.ts +++ b/apps/memos-local-plugin/core/session/types.ts @@ -193,8 +193,6 @@ export interface RelationInput { * is "scoring whether to terminate prevEpisodeId". */ prevEpisodeId?: EpisodeId; - /** Foreground request cancellation propagated to LLM classification. */ - signal?: AbortSignal; } // ─── Event bus ────────────────────────────────────────────────────────────── diff --git a/apps/memos-local-plugin/core/util/foreground-resources.ts b/apps/memos-local-plugin/core/util/foreground-resources.ts deleted file mode 100644 index d818d189f..000000000 --- a/apps/memos-local-plugin/core/util/foreground-resources.ts +++ /dev/null @@ -1,274 +0,0 @@ -import type { - EmbedCallOptions, - Embedder, - EmbedInput, -} from "../embedding/types.js"; -import type { EmbeddingVector } from "../types.js"; - -export type ResourcePriority = "foreground" | "background"; - -export interface ForegroundResources { - readonly shutdownSignal: AbortSignal; - /** Combine a request signal with the pipeline lifecycle signal. */ - signalFor(signal?: AbortSignal): AbortSignal; - /** Mark the complete turn.start path as foreground work. Idempotent release. */ - enterForeground(): () => void; - /** Background LLM work waits here before acquiring its existing semaphore. */ - waitForBackground(signal?: AbortSignal): Promise; - /** Priority-aware, non-preemptive embedding admission. */ - acquireEmbedding( - priority: ResourcePriority, - signal?: AbortSignal, - ): Promise<() => void>; - /** Reject queued work and cancel provider calls before pipeline drain. */ - shutdown(reason?: string): void; -} - -export interface ForegroundResourceOptions { - embeddingConcurrency?: number; - /** Prevent background starvation during a sustained foreground stream. */ - maxForegroundBurst?: number; -} - -interface Waiter { - resolve: (release: () => void) => void; - reject: (error: Error) => void; - signal?: AbortSignal; - onAbort?: () => void; -} - -interface BackgroundWaiter { - resolve: () => void; - reject: (error: Error) => void; - signal?: AbortSignal; - onAbort?: () => void; -} - -export function createForegroundResources( - options: ForegroundResourceOptions = {}, -): ForegroundResources { - const capacity = Math.max(1, Math.floor(options.embeddingConcurrency ?? 1)); - const maxForegroundBurst = Math.max( - 1, - Math.floor(options.maxForegroundBurst ?? 8), - ); - const embeddingWaiters: Record = { - foreground: [], - background: [], - }; - const backgroundWaiters: BackgroundWaiter[] = []; - let embeddingInUse = 0; - let foregroundActive = 0; - let foregroundBurst = 0; - const shutdownController = new AbortController(); - - function signalFor(signal?: AbortSignal): AbortSignal { - return signal - ? AbortSignal.any([signal, shutdownController.signal]) - : shutdownController.signal; - } - - function abortError(signal?: AbortSignal): Error { - return signal?.reason instanceof Error - ? signal.reason - : new DOMException("resource wait aborted", "AbortError"); - } - - function removeAbortListener(waiter: Waiter | BackgroundWaiter): void { - if (waiter.signal && waiter.onAbort) { - waiter.signal.removeEventListener("abort", waiter.onAbort); - } - } - - function nextEmbeddingWaiter(): { - priority: ResourcePriority; - waiter: Waiter; - } | null { - const foreground = embeddingWaiters.foreground; - const background = embeddingWaiters.background; - if ( - background.length > 0 && - (foreground.length === 0 || foregroundBurst >= maxForegroundBurst) - ) { - return { priority: "background", waiter: background.shift()! }; - } - if (foreground.length > 0) { - return { priority: "foreground", waiter: foreground.shift()! }; - } - if (background.length > 0) { - return { priority: "background", waiter: background.shift()! }; - } - return null; - } - - function drainEmbedding(): void { - while (embeddingInUse < capacity) { - const next = nextEmbeddingWaiter(); - if (!next) return; - removeAbortListener(next.waiter); - embeddingInUse++; - foregroundBurst = next.priority === "foreground" ? foregroundBurst + 1 : 0; - next.waiter.resolve(makeEmbeddingRelease()); - } - } - - function makeEmbeddingRelease(): () => void { - let released = false; - return (): void => { - if (released) return; - released = true; - embeddingInUse--; - drainEmbedding(); - }; - } - - function acquireEmbedding( - priority: ResourcePriority, - signal?: AbortSignal, - ): Promise<() => void> { - signal = signalFor(signal); - if (signal.aborted) return Promise.reject(abortError(signal)); - return new Promise((resolve, reject) => { - const waiter: Waiter = { resolve, reject, signal }; - if (signal) { - waiter.onAbort = () => { - const queue = embeddingWaiters[priority]; - const index = queue.indexOf(waiter); - if (index >= 0) queue.splice(index, 1); - reject(abortError(signal)); - }; - signal.addEventListener("abort", waiter.onAbort, { once: true }); - } - embeddingWaiters[priority].push(waiter); - drainEmbedding(); - }); - } - - function drainBackgroundGate(): void { - if (foregroundActive > 0) return; - for (const waiter of backgroundWaiters.splice(0)) { - removeAbortListener(waiter); - waiter.resolve(); - } - } - - function enterForeground(): () => void { - foregroundActive++; - let left = false; - return (): void => { - if (left) return; - left = true; - foregroundActive--; - drainBackgroundGate(); - }; - } - - function waitForBackground(signal?: AbortSignal): Promise { - signal = signalFor(signal); - if (signal.aborted) return Promise.reject(abortError(signal)); - if (foregroundActive === 0) return Promise.resolve(); - return new Promise((resolve, reject) => { - const waiter: BackgroundWaiter = { resolve, reject, signal }; - if (signal) { - waiter.onAbort = () => { - const index = backgroundWaiters.indexOf(waiter); - if (index >= 0) backgroundWaiters.splice(index, 1); - reject(abortError(signal)); - }; - signal.addEventListener("abort", waiter.onAbort, { once: true }); - } - backgroundWaiters.push(waiter); - }); - } - - function shutdown(reason = "pipeline shutdown"): void { - if (shutdownController.signal.aborted) return; - shutdownController.abort(new DOMException(reason, "AbortError")); - } - - return { - shutdownSignal: shutdownController.signal, - signalFor, - enterForeground, - waitForBackground, - acquireEmbedding, - shutdown, - }; -} - -/** - * Keep the Embedder contract intact while moving provider round-trips behind - * the shared priority arbiter. Background batches are deliberately chunked - * so one enrichment pass cannot monopolize the provider for an entire queue. - */ -export function prioritizeEmbedder( - inner: Embedder | null, - resources: ForegroundResources, - priority: ResourcePriority, - backgroundChunkSize = 8, -): Embedder | null { - if (!inner) return null; - - async function embedOne( - input: string | EmbedInput, - options?: EmbedCallOptions, - ): Promise { - const signal = resources.signalFor(options?.signal); - const callOptions = { ...options, signal }; - if (priority === "background") await resources.waitForBackground(signal); - const release = await resources.acquireEmbedding(priority, signal); - try { - return await inner!.embedOne(input, callOptions); - } finally { - release(); - } - } - - async function embedMany( - inputs: Array, - options?: EmbedCallOptions, - ): Promise { - const signal = resources.signalFor(options?.signal); - const callOptions = { ...options, signal }; - if (priority === "foreground" || inputs.length <= backgroundChunkSize) { - if (priority === "background") await resources.waitForBackground(signal); - const release = await resources.acquireEmbedding(priority, signal); - try { - return await inner!.embedMany(inputs, callOptions); - } finally { - release(); - } - } - - const results: EmbeddingVector[] = []; - for (let start = 0; start < inputs.length; start += backgroundChunkSize) { - await resources.waitForBackground(signal); - const release = await resources.acquireEmbedding(priority, signal); - try { - results.push( - ...await inner!.embedMany(inputs.slice(start, start + backgroundChunkSize), callOptions), - ); - } finally { - release(); - } - } - return results; - } - - return { - get dimensions() { - return inner.dimensions; - }, - get provider() { - return inner.provider; - }, - get model() { - return inner.model; - }, - embedOne, - embedMany, - stats: () => inner.stats(), - resetCache: () => inner.resetCache(), - close: () => inner.close(), - }; -} diff --git a/apps/memos-local-plugin/core/util/rate-limited-llm.ts b/apps/memos-local-plugin/core/util/rate-limited-llm.ts index 5bcb863bb..4d229b4c9 100644 --- a/apps/memos-local-plugin/core/util/rate-limited-llm.ts +++ b/apps/memos-local-plugin/core/util/rate-limited-llm.ts @@ -10,26 +10,20 @@ import type { LlmStreamChunk, } from "../llm/types.js"; import type { Semaphore } from "./semaphore.js"; -import type { ForegroundResources } from "./foreground-resources.js"; /** * Wrap an LLM client so expensive background subscribers share one * process-wide concurrency budget without changing call-site semantics. */ -export function rateLimitLlmClient( - client: LlmClient | null, - semaphore: Semaphore, - resources?: ForegroundResources, -): LlmClient | null { +export function rateLimitLlmClient(client: LlmClient | null, semaphore: Semaphore): LlmClient | null { if (!client) return null; - return new RateLimitedLlmClient(client, semaphore, resources); + return new RateLimitedLlmClient(client, semaphore); } class RateLimitedLlmClient implements LlmClient { constructor( private readonly inner: LlmClient, private readonly semaphore: Semaphore, - private readonly resources?: ForegroundResources, ) {} get provider(): LlmProviderName { @@ -48,12 +42,9 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCallOptions, ): Promise { - const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; - const callOpts = signal ? { ...opts, signal } : opts; - await this.resources?.waitForBackground(signal); - const release = await this.semaphore.acquire(signal); + const release = await this.semaphore.acquire(); try { - return await this.inner.complete(messages, callOpts); + return await this.inner.complete(messages, opts); } finally { release(); } @@ -63,12 +54,9 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCompleteJsonOptions, ): Promise> { - const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; - const callOpts = signal ? { ...opts, signal } : opts; - await this.resources?.waitForBackground(signal); - const release = await this.semaphore.acquire(signal); + const release = await this.semaphore.acquire(); try { - return await this.inner.completeJson(messages, callOpts); + return await this.inner.completeJson(messages, opts); } finally { release(); } @@ -78,12 +66,9 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCallOptions, ): AsyncIterable { - const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; - const callOpts = signal ? { ...opts, signal } : opts; - await this.resources?.waitForBackground(signal); - const release = await this.semaphore.acquire(signal); + const release = await this.semaphore.acquire(); try { - yield* this.inner.stream(messages, callOpts); + yield* this.inner.stream(messages, opts); } finally { release(); } diff --git a/apps/memos-local-plugin/core/util/request-deadline.ts b/apps/memos-local-plugin/core/util/request-deadline.ts deleted file mode 100644 index a7fb816a0..000000000 --- a/apps/memos-local-plugin/core/util/request-deadline.ts +++ /dev/null @@ -1,37 +0,0 @@ -export interface RequestDeadline { - readonly signal: AbortSignal; - remainingMs(): number; - dispose(): void; -} - -/** - * Convert an adapter-provided absolute epoch deadline into one abort signal. - * The absolute form survives JSON-RPC transport time and prevents every stage - * from accidentally receiving a fresh timeout budget. - */ -export function createRequestDeadline( - deadlineAt: number, - now: () => number = Date.now, -): RequestDeadline { - const controller = new AbortController(); - const remainingMs = (): number => Math.max(0, deadlineAt - now()); - const initialRemaining = remainingMs(); - let timer: ReturnType | null = null; - - if (!Number.isFinite(deadlineAt) || initialRemaining <= 0) { - controller.abort(new DOMException("request deadline exceeded", "TimeoutError")); - } else { - timer = setTimeout(() => { - controller.abort(new DOMException("request deadline exceeded", "TimeoutError")); - }, initialRemaining); - } - - return { - signal: controller.signal, - remainingMs, - dispose(): void { - if (timer) clearTimeout(timer); - timer = null; - }, - }; -} diff --git a/apps/memos-local-plugin/core/util/retry-after.ts b/apps/memos-local-plugin/core/util/retry-after.ts deleted file mode 100644 index e580e5f7b..000000000 --- a/apps/memos-local-plugin/core/util/retry-after.ts +++ /dev/null @@ -1,200 +0,0 @@ -/** Parse RFC 9110 Retry-After delay-seconds or HTTP-date into milliseconds. */ -export const MAX_INLINE_RETRY_DELAY_MS = 30_000; -/** @deprecated Use MAX_INLINE_RETRY_DELAY_MS. */ -export const MAX_RETRY_DELAY_MS = MAX_INLINE_RETRY_DELAY_MS; - -export type RetryDeferReason = - | "deadline_insufficient" - | "retry_after_too_long"; - -export interface RetryPlanBase { - backoffMs: number; - delayMs: number; - retryAfterMs: number | null; - retryAt: number; - source: "backoff" | "retry_after"; -} - -export type RetryPlan = - | (RetryPlanBase & { action: "wait" }) - | (RetryPlanBase & { action: "defer"; reason: RetryDeferReason }); - -export interface RetryCooldown { - retryAfterMs: number; - retryAt: number; - status: number; -} - -export interface RetryDiagnosticDetails { - retryAfterMs?: number; - retryAt?: number; - retryDecision?: "wait" | "defer" | "stop"; - retryReason?: string; -} - -const retryCooldowns = new Map(); - -export function parseRetryAfterMs( - value: string | null | undefined, - nowMs: number = Date.now(), -): number | null { - const raw = value?.trim(); - if (!raw) return null; - if (/^\d+$/.test(raw)) { - const seconds = Number(raw); - const delayMs = seconds * 1_000; - return Number.isSafeInteger(seconds) && Number.isSafeInteger(delayMs) - ? delayMs - : null; - } - // Retry-After only permits IMF-fixdate here. Keeping the shape strict avoids - // JavaScript accepting ambiguous strings such as "1.5" as a legacy date. - if (!/^[A-Za-z]{3}, \d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(raw)) return null; - const at = Date.parse(raw); - if (!Number.isFinite(at)) return null; - return Math.max(0, at - nowMs); -} - -export function retryDelayMs(input: { - attempt: number; - baseMs: number; - jitterMaxMs: number; - retryAfterMs?: number | null; - maxDelayMs?: number; - random?: () => number; -}): number { - const plan = planRetry({ - ...input, - maxInlineDelayMs: input.maxDelayMs, - }); - return plan.delayMs; -} - -/** - * Decide whether a retry can happen inline without violating Retry-After. - * - * Provider Retry-After values are never clamped downward. When the earliest - * legal retry cannot fit the inline wait or request deadline, callers must - * defer/fallback and carry retryAt into their recovery path. - */ -export function planRetry(input: { - attempt: number; - baseMs: number; - jitterMaxMs: number; - retryAfterMs?: number | null; - maxInlineDelayMs?: number; - deadlineAt?: number; - nowMs?: number; - random?: () => number; -}): RetryPlan { - const nowMs = input.nowMs ?? Date.now(); - const random = input.random ?? Math.random; - const jitter = Math.floor(random() * input.jitterMaxMs); - const rawBackoff = input.baseMs * 2 ** Math.max(0, input.attempt - 1) + jitter; - const maxInlineDelayMs = input.maxInlineDelayMs ?? MAX_INLINE_RETRY_DELAY_MS; - const backoffMs = Math.min(rawBackoff, maxInlineDelayMs); - const retryAfterMs = input.retryAfterMs ?? null; - const delayMs = Math.max(backoffMs, retryAfterMs ?? 0); - const retryAt = nowMs + delayMs; - const source = retryAfterMs !== null && retryAfterMs >= backoffMs - ? "retry_after" as const - : "backoff" as const; - const base: RetryPlanBase = { - backoffMs, - delayMs, - retryAfterMs, - retryAt, - source, - }; - - if (retryAfterMs !== null && retryAfterMs > maxInlineDelayMs) { - return { ...base, action: "defer", reason: "retry_after_too_long" }; - } - if (input.deadlineAt !== undefined && retryAt > input.deadlineAt) { - return { ...base, action: "defer", reason: "deadline_insufficient" }; - } - return { ...base, action: "wait" }; -} - -export function retryCooldownKey( - kind: "llm" | "embedding", - provider: string, - url: string, - scope: string = "", -): string { - return `${kind}\u0000${provider}\u0000${url}\u0000${scope}`; -} - -/** Extend a provider cooldown monotonically; a shorter later response cannot weaken it. */ -export function recordRetryCooldown(key: string, cooldown: RetryCooldown): void { - const current = retryCooldowns.get(key); - if (!current || cooldown.retryAt > current.retryAt) { - retryCooldowns.set(key, { ...cooldown }); - } -} - -export function getRetryCooldown( - key: string, - nowMs: number = Date.now(), -): RetryCooldown | null { - const cooldown = retryCooldowns.get(key); - if (!cooldown) return null; - if (cooldown.retryAt <= nowMs) { - retryCooldowns.delete(key); - return null; - } - return { ...cooldown }; -} - -/** Test/runtime-reset hook; plugin shutdown does not need to await cooldown state. */ -export function clearRetryCooldowns(): void { - retryCooldowns.clear(); -} - -/** Copy only bounded, machine-readable retry fields from an error detail bag. */ -export function extractRetryDiagnostics( - details: Record | undefined, -): RetryDiagnosticDetails { - if (!details) return {}; - const diagnostic: RetryDiagnosticDetails = {}; - if (typeof details.retryAfterMs === "number" && Number.isFinite(details.retryAfterMs)) { - diagnostic.retryAfterMs = details.retryAfterMs; - } - if (typeof details.retryAt === "number" && Number.isFinite(details.retryAt)) { - diagnostic.retryAt = details.retryAt; - } - if ( - details.retryDecision === "wait" - || details.retryDecision === "defer" - || details.retryDecision === "stop" - ) { - diagnostic.retryDecision = details.retryDecision; - } - if (typeof details.retryReason === "string") { - diagnostic.retryReason = details.retryReason; - } - return diagnostic; -} - -/** Abortable retry wait so request cancellation and shutdown do not leave sleepers behind. */ -export function waitForRetry(delayMs: number, signal?: AbortSignal): Promise { - if (signal?.aborted) return Promise.reject(abortReason(signal)); - if (delayMs <= 0) return Promise.resolve(); - - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { - signal?.removeEventListener("abort", onAbort); - resolve(); - }, delayMs); - const onAbort = () => { - clearTimeout(timer); - signal?.removeEventListener("abort", onAbort); - reject(signal ? abortReason(signal) : new DOMException("Aborted", "AbortError")); - }; - signal?.addEventListener("abort", onAbort, { once: true }); - }); -} - -function abortReason(signal: AbortSignal): unknown { - return signal.reason ?? new DOMException("Aborted", "AbortError"); -} diff --git a/apps/memos-local-plugin/core/util/semaphore.ts b/apps/memos-local-plugin/core/util/semaphore.ts index 037db2607..8dd9b75fc 100644 --- a/apps/memos-local-plugin/core/util/semaphore.ts +++ b/apps/memos-local-plugin/core/util/semaphore.ts @@ -1,37 +1,23 @@ export interface Semaphore { - acquire(signal?: AbortSignal): Promise<() => void>; -} - -interface Waiter { - resolve: (release: () => void) => void; - reject: (error: Error) => void; - signal?: AbortSignal; - onAbort?: () => void; + acquire(): Promise<() => void>; } export function createSemaphore(max: number): Semaphore { const limit = Math.max(1, Math.floor(max)); let current = 0; - const waiters: Waiter[] = []; + const waiters: Array<() => void> = []; return { - async acquire(signal?: AbortSignal) { - if (signal?.aborted) throw abortError(signal); + async acquire() { if (current < limit) { current++; return release; } - return new Promise<() => void>((resolve, reject) => { - const waiter: Waiter = { resolve, reject, signal }; - if (signal) { - waiter.onAbort = () => { - const index = waiters.indexOf(waiter); - if (index >= 0) waiters.splice(index, 1); - reject(abortError(signal)); - }; - signal.addEventListener("abort", waiter.onAbort, { once: true }); - } - waiters.push(waiter); + return new Promise<() => void>((resolve) => { + waiters.push(() => { + current++; + resolve(release); + }); }); }, }; @@ -39,17 +25,6 @@ export function createSemaphore(max: number): Semaphore { function release() { current = Math.max(0, current - 1); const next = waiters.shift(); - if (!next) return; - if (next.signal && next.onAbort) { - next.signal.removeEventListener("abort", next.onAbort); - } - current++; - next.resolve(release); + if (next) next(); } } - -function abortError(signal: AbortSignal): Error { - return signal.reason instanceof Error - ? signal.reason - : new DOMException("semaphore wait aborted", "AbortError"); -} diff --git a/apps/memos-local-plugin/tests/python/test_bridge_client.py b/apps/memos-local-plugin/tests/python/test_bridge_client.py index b5c9eb917..82763fcea 100644 --- a/apps/memos-local-plugin/tests/python/test_bridge_client.py +++ b/apps/memos-local-plugin/tests/python/test_bridge_client.py @@ -448,128 +448,6 @@ def test_reverse_request_waits_for_late_host_handler_registration(self) -> None: self.assertNotIn("error", response) client.close() - def test_slow_reverse_handler_does_not_block_regular_rpc_responses(self) -> None: - """A host LLM callback must not stall the stdout response demux. - - ``host.llm.complete`` can legitimately spend several seconds in the - Hermes model client. The bridge reader still has to resolve an - unrelated foreground ``turn.start`` response during that - window; otherwise one background callback head-of-line blocks every - provider lease sharing the process. - """ - client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") - assert self._fake is not None - handler_started = threading.Event() - release_handler = threading.Event() - - def _slow_handler(_params: dict) -> dict: - handler_started.set() - release_handler.wait(timeout=2.0) - return {"text": "host:done", "model": "host-test"} - - client.register_host_handler("host.llm.complete", _slow_handler) - self._fake.stdout._enqueue( - { - "jsonrpc": "2.0", - "id": "srv-slow", - "method": "host.llm.complete", - "params": {"messages": [{"role": "user", "content": "slow"}]}, - } - ) - self.assertTrue(handler_started.wait(timeout=0.5)) - - try: - response = client.request( - "turn.start", - { - "sessionId": "hermes:session:1", - "userText": "foreground recall", - }, - timeout=0.5, - ) - self.assertIn("foreground recall", response["injectedContext"]) - finally: - release_handler.set() - - reverse_response = self._wait_for_client_write(lambda msg: msg.get("id") == "srv-slow") - self.assertEqual(reverse_response["result"]["text"], "host:done") - client.close() - - def test_reverse_handler_queue_rejects_overload_without_blocking_reader(self) -> None: - client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") - assert self._fake is not None - handler_started = threading.Event() - release_handler = threading.Event() - - def _slow_handler(_params: dict) -> dict: - handler_started.set() - release_handler.wait(timeout=2.0) - return {"text": "done"} - - client.register_host_handler("host.llm.complete", _slow_handler) - self._fake.stdout._enqueue( - { - "jsonrpc": "2.0", - "id": "srv-running", - "method": "host.llm.complete", - "params": {}, - } - ) - self.assertTrue(handler_started.wait(timeout=0.5)) - - overflow_id = "srv-overflow" - for index in range(bridge_client_mod.HOST_HANDLER_QUEUE_CAPACITY + 1): - rpc_id = ( - overflow_id - if index == bridge_client_mod.HOST_HANDLER_QUEUE_CAPACITY - else f"srv-{index}" - ) - self._fake.stdout._enqueue( - { - "jsonrpc": "2.0", - "id": rpc_id, - "method": "host.llm.complete", - "params": {}, - } - ) - - try: - response = self._wait_for_client_write(lambda msg: msg.get("id") == overflow_id) - self.assertEqual(response["error"]["data"]["code"], "host_handler_busy") - finally: - client.close() - release_handler.set() - - def test_close_does_not_wait_for_a_running_reverse_handler(self) -> None: - """An uncooperative host callback must not extend bridge shutdown.""" - client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") - assert self._fake is not None - handler_started = threading.Event() - release_handler = threading.Event() - - def _slow_handler(_params: dict) -> dict: - handler_started.set() - release_handler.wait(timeout=2.0) - return {"text": "late", "model": "host-test"} - - client.register_host_handler("host.llm.complete", _slow_handler) - self._fake.stdout._enqueue( - { - "jsonrpc": "2.0", - "id": "srv-close", - "method": "host.llm.complete", - "params": {}, - } - ) - self.assertTrue(handler_started.wait(timeout=0.5)) - - started = time.monotonic() - try: - client.close() - finally: - release_handler.set() - self.assertLess(time.monotonic() - started, 0.5) - def test_reader_exit_marks_pending_as_transport_closed(self) -> None: """R1 (#2028): reader thread EOF must wake pending waiters with transport_closed instead of leaving them parked on their @@ -1233,7 +1111,7 @@ def test_sync_turn_uses_long_rpc_timeout_for_turn_end(self) -> None: "sessions (issue #2028).", ) - def test_prefetch_uses_dedicated_foreground_timeout_for_turn_start(self) -> None: + def test_prefetch_uses_long_rpc_timeout_for_turn_start(self) -> None: p = self._provider_mod.MemTensorProvider() bridge = RecordingBridge() p._bridge = bridge @@ -1244,45 +1122,12 @@ def test_prefetch_uses_dedicated_foreground_timeout_for_turn_start(self) -> None self.assertIn("turn.start", methods) start_index = methods.index("turn.start") start_kwargs = bridge.call_kwargs[start_index] - self.assertGreater(start_kwargs.get("timeout", 0.0), 0.0) - self.assertLessEqual( + self.assertGreaterEqual( start_kwargs.get("timeout", 0.0), - self._provider_mod._PREFETCH_RPC_TIMEOUT, - "foreground turn.start must finish before the Hermes host deadline; " - "long capture work keeps the separate issue #2028 timeout.", + self._EXPECTED_LONG_TIMEOUT, + "turn.start suffers the same long-tail latency as turn.end and " + "must share the long RPC timeout (issue #2028).", ) - start_payload = bridge.calls[start_index][1] - self.assertIn("deadlineAt", start_payload) - self.assertGreater(start_payload["deadlineAt"], start_payload["ts"]) - - def test_foreground_reconnect_and_retry_share_one_deadline(self) -> None: - class ClosedBridge: - def request(self, *_args, **_kwargs) -> dict: - raise BridgeError("transport_closed", "bridge closed") - - p = self._provider_mod.MemTensorProvider() - p._bridge = ClosedBridge() - recovered = RecordingBridge() - monotonic_now = [100.0] - - def reconnect(_session_id: str, *, timeout: float) -> None: - self.assertLessEqual(timeout, 6.0) - monotonic_now[0] += 4.0 - p._bridge = recovered - - with ( - patch("memos_provider.time.monotonic", side_effect=lambda: monotonic_now[0]), - patch.object(p, "_reconnect_bridge", side_effect=reconnect), - ): - p._bridge_request_with_retry( - "turn.start", - {"sessionId": "s-1"}, - timeout=6.0, - deadline_monotonic=106.0, - ) - - self.assertEqual(recovered.calls[0][0], "turn.start") - self.assertLessEqual(recovered.call_kwargs[0]["timeout"], 2.0) class ViewerDaemonTests(unittest.TestCase): diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index ecd0e1ae1..6b95ae61b 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -480,75 +480,6 @@ def test_prefetch_passes_stable_turn_key_to_bridge(self) -> None: turn_start = next(params for method, params in bridge.calls if method == "turn.start") self.assertEqual(turn_start["turnKey"], "turn-key-session:7") - def test_prefetch_uses_a_dedicated_budget_and_forwards_absolute_deadline(self) -> None: - bridge = FakeBridge() - with ( - patch("memos_provider.ensure_bridge_running", return_value=True), - patch("memos_provider.ensure_viewer_daemon", return_value=True), - patch("memos_provider.MemosBridgeClient", return_value=bridge), - patch("memos_provider._PREFETCH_RPC_TIMEOUT", 6.0), - patch("memos_provider.time.time", return_value=1_700_000_000.0), - ): - provider = memos_provider.MemTensorProvider() - provider.initialize("budget-session") - provider.on_turn_start(1, "recall the build decision") - with patch.object( - provider, - "_bridge_request_with_retry", - wraps=provider._bridge_request_with_retry, - ) as request: - provider.prefetch("recall the build decision") - - turn_start = next(params for method, params in bridge.calls if method == "turn.start") - self.assertEqual(turn_start["deadlineAt"], 1_700_000_005_750) - request.assert_called_once() - self.assertLessEqual(request.call_args.kwargs["timeout"], 6.0) - self.assertIn("deadline_monotonic", request.call_args.kwargs) - - def test_prefetch_budget_includes_bridge_ensure_time(self) -> None: - bridge = FakeBridge() - monotonic_now = [100.0] - - def ensure_bridge(_session_id: str, *, timeout: float) -> bool: - self.assertAlmostEqual(timeout, 6.0, places=3) - monotonic_now[0] += 2.5 - return True - - with ( - patch("memos_provider.ensure_bridge_running", return_value=True), - patch("memos_provider.ensure_viewer_daemon", return_value=True), - patch("memos_provider.MemosBridgeClient", return_value=bridge), - patch("memos_provider._PREFETCH_RPC_TIMEOUT", 6.0), - patch("memos_provider.time.time", return_value=1_700_000_000.0), - patch("memos_provider.time.monotonic", side_effect=lambda: monotonic_now[0]), - ): - provider = memos_provider.MemTensorProvider() - provider.initialize("budget-session") - provider.on_turn_start(1, "recall the build decision") - with ( - patch.object(provider, "_ensure_bridge", side_effect=ensure_bridge), - patch.object( - provider, - "_bridge_request_with_retry", - wraps=provider._bridge_request_with_retry, - ) as request, - ): - provider.prefetch("recall the build decision") - - self.assertLessEqual(request.call_args.kwargs["timeout"], 3.5) - turn_start = next(params for method, params in bridge.calls if method == "turn.start") - self.assertEqual(turn_start["deadlineAt"], 1_700_000_005_750) - - def test_prefetch_timeout_config_rejects_non_positive_values(self) -> None: - with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "0"}): - self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 6.0) - with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "nan"}): - self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 6.0) - with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "4.5"}): - self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 4.5) - with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "30"}): - self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 7.0) - def test_prefetch_suppresses_memory_injection_for_explicit_delegation(self) -> None: bridge = FakeBridge() with ( diff --git a/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts b/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts index fc9f921e1..8ee5fa921 100644 --- a/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts +++ b/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts @@ -344,17 +344,6 @@ describe("makeDispatcher", () => { ).rejects.toSatisfy( (err) => err instanceof MemosError && err.code === "invalid_argument", ); - await expect( - dispatch("turn.start", { - agent: "openclaw", - sessionId: "s-1", - userText: "hi", - ts: 123, - deadlineAt: "soon", - }), - ).rejects.toSatisfy( - (err) => err instanceof MemosError && err.code === "invalid_argument", - ); }); it("feedback.submit forwards the DTO shape intact", async () => { diff --git a/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts b/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts index 617bc4a72..3c608a53a 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts @@ -6,10 +6,8 @@ import { initTestLogger } from "../../../core/logger/index.js"; import type { EmbedRole, EmbeddingConfig, - EmbeddingErrorDetail, EmbeddingProvider, EmbeddingProviderName, - EmbeddingStatusDetail, ProviderCallCtx, } from "../../../core/embedding/types.js"; @@ -77,24 +75,6 @@ describe("embedder facade", () => { expect(Array.from(v)).toEqual([3, 97, 0]); // a=97 }); - it("forwards the caller abort signal and deadline to the provider", async () => { - const seen: Array> = []; - const p: EmbeddingProvider = { - name: "openai_compatible", - async embed(texts, _role, ctx) { - seen.push({ signal: ctx.signal, deadlineAt: ctx.deadlineAt }); - return texts.map(() => [1, 2, 3]); - }, - }; - const e = createEmbedderWithProvider(cfg(), p); - const controller = new AbortController(); - - const deadlineAt = Date.now() + 1_000; - await e.embedOne("signal", { signal: controller.signal, deadlineAt }); - - expect(seen).toEqual([{ signal: controller.signal, deadlineAt }]); - }); - it("dedups identical inputs into one provider call", async () => { const p = new FakeProvider(); const e = createEmbedderWithProvider(cfg(), p); @@ -195,45 +175,6 @@ describe("embedder facade", () => { } }); - it("preserves deferred retry diagnostics in error and status sinks", async () => { - const errors: EmbeddingErrorDetail[] = []; - const statuses: EmbeddingStatusDetail[] = []; - const retryAt = Date.now() + 120_000; - const provider: EmbeddingProvider = { - name: "openai_compatible", - async embed() { - throw new MemosError("embedding_unavailable", "provider cooldown", { - retryAfterMs: 120_000, - retryAt, - retryDecision: "defer", - retryReason: "retry_after_too_long", - }); - }, - }; - const e = createEmbedderWithProvider( - cfg({ onError: (detail) => errors.push(detail), onStatus: (detail) => statuses.push(detail) }), - provider, - ); - - await expect(e.embedOne("x")).rejects.toBeInstanceOf(MemosError); - - expect(errors).toContainEqual( - expect.objectContaining({ - retryAfterMs: 120_000, - retryAt, - retryDecision: "defer", - retryReason: "retry_after_too_long", - }), - ); - expect(statuses).toContainEqual( - expect.objectContaining({ - status: "error", - retryAt, - retryDecision: "defer", - }), - ); - }); - it("rejects when provider returns too few rows", async () => { const e = createEmbedderWithProvider(cfg({ provider: "gemini" }), new WrongCountProvider()); await expect(e.embedMany(["x", "y", "z"])).rejects.toBeInstanceOf(MemosError); diff --git a/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts b/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts index 2811e94ef..3b9ee5bf2 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts @@ -4,7 +4,6 @@ import { MemosError } from "../../../agent-contract/errors.js"; import { initTestLogger } from "../../../core/logger/index.js"; import { httpPostJson } from "../../../core/embedding/fetcher.js"; import type { ProviderLogger } from "../../../core/embedding/types.js"; -import { clearRetryCooldowns } from "../../../core/util/retry-after.js"; function nullLogger(): ProviderLogger { return { @@ -22,8 +21,6 @@ describe("embedding/fetcher", () => { vi.useRealTimers(); // retry backoff uses real setTimeout; keep it real but short }); afterEach(() => { - clearRetryCooldowns(); - vi.useRealTimers(); vi.unstubAllGlobals(); }); @@ -84,85 +81,6 @@ describe("embedding/fetcher", () => { expect(f).toHaveBeenCalledTimes(2); }); - it("honors Retry-After HTTP-date before retrying a 429", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); - const f = mockFetch([ - new Response("rate limited", { - status: 429, - headers: { "Retry-After": "Tue, 04 Aug 2026 00:00:03 GMT" }, - }), - new Response(JSON.stringify({ ok: 1 }), { status: 200 }), - ]); - - const pending = httpPostJson<{ ok: number }>({ - url: "https://x", - body: {}, - provider: "cohere", - log: nullLogger(), - maxRetries: 1, - }); - await vi.advanceTimersByTimeAsync(2_999); - expect(f).toHaveBeenCalledTimes(1); - await vi.advanceTimersByTimeAsync(1); - await expect(pending).resolves.toEqual({ ok: 1 }); - expect(f).toHaveBeenCalledTimes(2); - vi.useRealTimers(); - }); - - it("defers a long Retry-After and short-circuits the provider cooldown", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); - const f = mockFetch([ - new Response("maintenance", { status: 503, headers: { "Retry-After": "120" } }), - ]); - const opts = { - url: "https://embedding-x", - body: {}, - provider: "cohere" as const, - log: nullLogger(), - maxRetries: 1, - }; - - await expect(httpPostJson(opts)).rejects.toMatchObject({ - code: "embedding_unavailable", - details: { - retryAfterMs: 120_000, - retryDecision: "defer", - retryReason: "retry_after_too_long", - }, - }); - await expect(httpPostJson(opts)).rejects.toMatchObject({ - code: "embedding_unavailable", - details: { retryReason: "cooldown_active" }, - }); - expect(f).toHaveBeenCalledTimes(1); - }); - - it("returns structured diagnostics when network backoff cannot fit the deadline", async () => { - vi.useFakeTimers(); - const now = Date.parse("2026-08-04T00:00:00.000Z"); - vi.setSystemTime(now); - const f = mockFetch([new Error("ECONNRESET")]); - - await expect(httpPostJson({ - url: "https://embedding-deadline", - body: {}, - provider: "mistral", - log: nullLogger(), - maxRetries: 1, - deadlineAt: now + 100, - })).rejects.toMatchObject({ - name: "MemosError", - code: "embedding_unavailable", - details: { - retryDecision: "defer", - retryReason: "deadline_insufficient", - }, - }); - expect(f).toHaveBeenCalledTimes(1); - }); - it("does not retry on 400", async () => { mockFetch([new Response("bad", { status: 400 })]); await expect( diff --git a/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts b/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts index ae72cc60e..f08a37f32 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts @@ -1,7 +1,6 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { createEmbeddingRetryWorker } from "../../../core/embedding/retry-worker.js"; -import { ERROR_CODES, MemosError } from "../../../agent-contract/errors.js"; import { rootLogger } from "../../../core/logger/index.js"; import type { EpisodeId, SessionId, TraceId } from "../../../core/types.js"; import { makeTmpDb, type TmpDbHandle } from "../../helpers/tmp-db.js"; @@ -175,39 +174,6 @@ describe("embedding retry worker", () => { expect(handle.repos.apiLogs.list({ toolName: "system_error", limit: 5, offset: 0 })).toHaveLength(1); }); - it("never schedules a durable retry before the provider retryAt", async () => { - const retryAt = NOW + 120_000; - handle.repos.embeddingRetryQueue.enqueue({ - id: "er_retry_after", - targetKind: "trace", - targetId: "tr_retry", - vectorField: "vec_summary", - sourceText: "retry me later", - maxAttempts: 3, - now: NOW, - }); - const worker = createEmbeddingRetryWorker({ - repos: handle.repos, - embedder: fakeEmbedder({ - throwWith: new MemosError( - ERROR_CODES.EMBEDDING_UNAVAILABLE, - "provider cooling down", - { retryAt, retryAfterMs: 120_000, retryDecision: "defer" }, - ), - }), - log: rootLogger.child({ channel: "test.embedding.retry" }), - now: () => NOW, - }); - - await worker.flush(); - - expect(queueRow(handle, "er_retry_after")).toMatchObject({ - status: "pending", - attempts: 1, - next_attempt_at: retryAt, - }); - }); - it("treats missing target rows as retry failures", async () => { handle.repos.embeddingRetryQueue.enqueue({ id: "er_missing", @@ -235,30 +201,4 @@ describe("embedding retry worker", () => { last_error: "embedding retry target not found: trace:tr_missing", }); }); - - it("does not claim new retry jobs after stop during shutdown", async () => { - handle.repos.embeddingRetryQueue.enqueue({ - id: "er_shutdown", - targetKind: "trace", - targetId: "tr_retry", - vectorField: "vec_summary", - sourceText: "do not start during shutdown", - now: NOW, - }); - const worker = createEmbeddingRetryWorker({ - repos: handle.repos, - embedder: fakeEmbedder({ dimensions: 8 }), - log: rootLogger.child({ channel: "test.embedding.retry" }), - now: () => NOW, - }); - - worker.stop(); - await worker.flush(); - - expect(queueRow(handle, "er_shutdown")).toMatchObject({ - status: "pending", - attempts: 0, - claimed_by: null, - }); - }); }); diff --git a/apps/memos-local-plugin/tests/unit/llm/client.test.ts b/apps/memos-local-plugin/tests/unit/llm/client.test.ts index cd125ecd8..dee0de228 100644 --- a/apps/memos-local-plugin/tests/unit/llm/client.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/client.test.ts @@ -283,45 +283,6 @@ describe("llm/client", () => { await expect(client.complete([] as LlmMessage[])).rejects.toBeInstanceOf(MemosError); }); - it("preserves deferred retry diagnostics in error and status sinks", async () => { - const errors: Array> = []; - const statuses: LlmStatusDetail[] = []; - const retryAt = Date.now() + 120_000; - const provider = new ThrowingProvider( - new MemosError(ERROR_CODES.LLM_RATE_LIMITED, "provider cooldown", { - retryAfterMs: 120_000, - retryAt, - retryDecision: "defer", - retryReason: "retry_after_too_long", - }), - ); - const client = createLlmClientWithProvider( - cfg({ - onError: (detail) => errors.push(detail as unknown as Record), - onStatus: (detail) => statuses.push(detail), - }), - provider, - ); - - await expect(client.complete("x")).rejects.toBeInstanceOf(MemosError); - - expect(errors).toContainEqual( - expect.objectContaining({ - retryAfterMs: 120_000, - retryAt, - retryDecision: "defer", - retryReason: "retry_after_too_long", - }), - ); - expect(statuses).toContainEqual( - expect.objectContaining({ - status: "error", - retryAt, - retryDecision: "defer", - }), - ); - }); - // ─── Circuit breaker (issue #1897) ────────────────────────────────────── describe("circuit breaker", () => { function statusSink(): { rows: LlmStatusDetail[]; push: (d: LlmStatusDetail) => void } { diff --git a/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts b/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts index 9b17e1b16..a3941b7aa 100644 --- a/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts @@ -4,7 +4,6 @@ import { MemosError } from "../../../agent-contract/errors.js"; import { decodeSse, httpPostJson, httpPostStream } from "../../../core/llm/fetcher.js"; import { initTestLogger } from "../../../core/logger/index.js"; import type { LlmProviderLogger } from "../../../core/llm/types.js"; -import { clearRetryCooldowns } from "../../../core/util/retry-after.js"; function nullLog(): LlmProviderLogger { return { @@ -30,205 +29,7 @@ function mockFetch(replies: Array) { describe("llm/fetcher", () => { beforeAll(() => initTestLogger()); - afterEach(() => { - clearRetryCooldowns(); - vi.useRealTimers(); - vi.unstubAllGlobals(); - }); - - it("honors Retry-After delay-seconds before retrying a 429", async () => { - vi.useFakeTimers(); - const f = mockFetch([ - new Response("slow down", { status: 429, headers: { "Retry-After": "2" } }), - new Response(JSON.stringify({ ok: 1 }), { status: 200 }), - ]); - - const pending = httpPostJson({ - url: "https://x", - body: {}, - timeoutMs: 5_000, - maxRetries: 1, - provider: "openai_compatible", - log: nullLog(), - }); - await vi.advanceTimersByTimeAsync(1_999); - expect(f).toHaveBeenCalledTimes(1); - await vi.advanceTimersByTimeAsync(1); - await expect(pending).resolves.toMatchObject({ json: { ok: 1 } }); - expect(f).toHaveBeenCalledTimes(2); - vi.useRealTimers(); - }); - - it("aborts while waiting for Retry-After", async () => { - vi.useFakeTimers(); - const ctrl = new AbortController(); - const f = mockFetch([ - new Response("slow down", { status: 429, headers: { "Retry-After": "2" } }), - ]); - - const pending = httpPostJson({ - url: "https://x", - body: {}, - timeoutMs: 5_000, - maxRetries: 1, - signal: ctrl.signal, - provider: "openai_compatible", - log: nullLog(), - }); - await vi.advanceTimersByTimeAsync(0); - ctrl.abort(); - await expect(pending).rejects.toBeInstanceOf(MemosError); - expect(f).toHaveBeenCalledTimes(1); - vi.useRealTimers(); - }); - - it("defers a long Retry-After from a 503 without retrying early", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); - const warn = vi.fn(); - const f = mockFetch([ - new Response("maintenance", { status: 503, headers: { "Retry-After": "120" } }), - ]); - - const pending = httpPostJson({ - url: "https://x", - body: {}, - timeoutMs: 120_000, - maxRetries: 1, - provider: "openai_compatible", - log: { ...nullLog(), warn }, - }); - await expect(pending).rejects.toMatchObject({ - code: "llm_unavailable", - details: { - retryAfterMs: 120_000, - retryDecision: "defer", - retryReason: "retry_after_too_long", - }, - }); - expect(f).toHaveBeenCalledTimes(1); - expect(warn).toHaveBeenCalledWith( - "http.retry_deferred", - expect.objectContaining({ - retryAfterMs: 120_000, - retryDecision: "defer", - retryReason: "retry_after_too_long", - }), - ); - }); - - it("short-circuits calls while the provider Retry-After cooldown is active", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); - const warn = vi.fn(); - const f = mockFetch([ - new Response("slow down", { status: 429, headers: { "Retry-After": "120" } }), - ]); - const opts = { - url: "https://x", - body: {}, - timeoutMs: 5_000, - maxRetries: 1, - provider: "openai_compatible" as const, - log: { ...nullLog(), warn }, - }; - - await expect(httpPostJson(opts)).rejects.toMatchObject({ code: "llm_rate_limited" }); - await expect(httpPostJson(opts)).rejects.toMatchObject({ - code: "llm_rate_limited", - details: { retryDecision: "defer", retryReason: "cooldown_active" }, - }); - expect(f).toHaveBeenCalledTimes(1); - expect(warn).toHaveBeenCalledWith( - "http.retry_cooldown", - expect.objectContaining({ retryReason: "cooldown_active" }), - ); - }); - - it("does not enter a Retry-After wait that cannot fit the absolute deadline", async () => { - vi.useFakeTimers(); - const now = Date.parse("2026-08-04T00:00:00.000Z"); - vi.setSystemTime(now); - const f = mockFetch([ - new Response("slow down", { status: 429, headers: { "Retry-After": "5" } }), - ]); - - await expect(httpPostJson({ - url: "https://deadline", - body: {}, - timeoutMs: 5_000, - maxRetries: 1, - deadlineAt: now + 1_000, - provider: "openai_compatible", - log: nullLog(), - })).rejects.toMatchObject({ - code: "llm_rate_limited", - details: { - retryDecision: "defer", - retryReason: "deadline_insufficient", - }, - }); - expect(f).toHaveBeenCalledTimes(1); - }); - - it("returns structured diagnostics when network backoff cannot fit the deadline", async () => { - vi.useFakeTimers(); - const now = Date.parse("2026-08-04T00:00:00.000Z"); - vi.setSystemTime(now); - const f = mockFetch([new Error("ECONNRESET")]); - - await expect(httpPostJson({ - url: "https://network-deadline", - body: {}, - timeoutMs: 5_000, - maxRetries: 1, - deadlineAt: now + 100, - provider: "openai_compatible", - log: nullLog(), - })).rejects.toMatchObject({ - name: "MemosError", - code: "llm_unavailable", - details: { - retryDecision: "defer", - retryReason: "deadline_insufficient", - }, - }); - expect(f).toHaveBeenCalledTimes(1); - }); - - it("does not let an older in-flight success clear a newer provider cooldown", async () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); - let resolveSuccess!: (response: Response) => void; - const success = new Promise((resolve) => { resolveSuccess = resolve; }); - const f = vi.fn() - .mockImplementationOnce(() => success) - .mockResolvedValueOnce( - new Response("slow down", { status: 429, headers: { "Retry-After": "120" } }), - ); - vi.stubGlobal("fetch", f); - const opts = { - url: "https://shared-endpoint", - body: {}, - timeoutMs: 5_000, - maxRetries: 1, - provider: "openai_compatible" as const, - log: nullLog(), - }; - - const older = httpPostJson<{ ok: boolean }>(opts); - await vi.waitFor(() => expect(f).toHaveBeenCalledTimes(1)); - await expect(httpPostJson(opts)).rejects.toMatchObject({ - details: { retryReason: "retry_after_too_long" }, - }); - resolveSuccess(new Response(JSON.stringify({ ok: true }), { status: 200 })); - await expect(older).resolves.toMatchObject({ json: { ok: true } }); - - await expect(httpPostJson(opts)).rejects.toMatchObject({ - details: { retryReason: "cooldown_active" }, - }); - expect(f).toHaveBeenCalledTimes(2); - }); + afterEach(() => vi.unstubAllGlobals()); it("returns parsed JSON on 200", async () => { mockFetch([new Response(JSON.stringify({ a: 1 }), { status: 200 })]); diff --git a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts index 52853ff4a..cb5140d09 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts @@ -898,53 +898,6 @@ describe("MemoryCore façade", () => { expect(scored.priority).toBe(1); }); - it("logs both user and assistant content with the matching role", async () => { - pipeline = createPipeline(buildDeps(db!)); - core = createMemoryCore( - pipeline, - resolveHome("openclaw", "/tmp/memos-mc-test"), - "test", - ); - await core.init(); - - const userText = "今晚吃什么，推荐一下"; - const agentText = "可以考虑清淡的汤面、盖饭或者附近评价不错的家常菜。"; - const start = await core.onTurnStart({ - agent: "openclaw", - sessionId: "s-memory-add-roles", - userText, - ts: 1_700_000_000_000, - }); - await core.onTurnEnd({ - agent: "openclaw", - sessionId: start.query.sessionId!, - episodeId: start.query.episodeId!, - agentText, - toolCalls: [], - ts: 1_700_000_000_500, - }); - - const { logs } = await core.listApiLogs({ - toolName: "memory_add", - limit: 10, - }); - const liteLog = logs.find((log) => { - const input = JSON.parse(log.inputJson) as { phase?: string }; - return input.phase === "lite"; - }); - expect(liteLog).toBeDefined(); - - const output = JSON.parse(liteLog!.outputJson) as { - details?: Array<{ role?: string; content?: string }>; - }; - expect( - output.details?.map(({ role, content }) => ({ role, content })), - ).toEqual([ - { role: "user", content: userText }, - { role: "assistant", content: agentText }, - ]); - }); - it("onTurnEnd preserves adapter-provided historical timestamps", async () => { pipeline = createPipeline(buildDeps(db!)); core = createMemoryCore( diff --git a/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts index 1e79c861a..a70fdcf6b 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts @@ -77,39 +77,6 @@ afterEach(async () => { }); describe("pipeline/orchestrator", () => { - it("degrades retrieval at the adapter deadline and aborts the provider call", async () => { - const base = fakeEmbedder({ dimensions: 384 }); - let sawAbort = false; - const embedder = { - ...base, - async embedOne(input: Parameters[0], options?: { signal?: AbortSignal }) { - return await new Promise>>((resolve, reject) => { - const onAbort = () => { - sawAbort = true; - reject(new DOMException("deadline", "AbortError")); - }; - if (options?.signal?.aborted) return onAbort(); - options?.signal?.addEventListener("abort", onAbort, { once: true }); - void resolve; - }); - }, - }; - pipeline = createPipeline(buildDeps(dbHandle!, embedder)); - const startedAt = Date.now(); - - const packet = await pipeline.onTurnStart({ - agent: "hermes", - sessionId: "s-deadline", - userText: "find the previous build decision", - ts: Date.now(), - deadlineAt: Date.now() + 25, - }); - - expect(sawAbort).toBe(true); - expect(Date.now() - startedAt).toBeLessThan(500); - expect(packet.reason).toBe("turn_start"); - }); - it("threads a dedicated l3Llm through to the handle", () => { const l3Llm = fakeLlm({ completeJson: {} }); pipeline = createPipeline({ ...buildDeps(dbHandle!), l3Llm }); diff --git a/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts b/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts index c3321635a..6b37ae55f 100644 --- a/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts +++ b/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts @@ -109,25 +109,6 @@ describe("session/intent-classifier", () => { expect(d.signals).toEqual(["llm"]); }); - it("forwards the foreground abort signal and classifier timeout", async () => { - let seen: { signal?: AbortSignal; timeoutMs?: number } | undefined; - const llm = fakeLlm(() => ({ kind: "task", confidence: 0.8, reason: "task" })); - const original = llm.completeJson.bind(llm); - llm.completeJson = async (messages, opts) => { - seen = opts; - return original(messages, opts); - }; - const controller = new AbortController(); - const c = createIntentClassifier({ llm, timeoutMs: 321 }); - - await c.classify("investigate an ambiguous pipeline issue", { - signal: controller.signal, - }); - - expect(seen?.signal).toBe(controller.signal); - expect(seen?.timeoutMs).toBe(321); - }); - it("LLM failure falls back to heuristic", async () => { const c = createIntentClassifier({ llm: fakeLlm(() => { diff --git a/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts b/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts index a17d3d9ce..575a4b44a 100644 --- a/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts +++ b/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts @@ -119,33 +119,6 @@ describe("relation-classifier — V7 §0.1", () => { expect(d.llmModel).toBe("fake/test-model"); }); - it("forwards the foreground abort signal and classifier timeout", async () => { - let seen: { signal?: AbortSignal; timeoutMs?: number } | undefined; - const controller = new AbortController(); - const c = createRelationClassifier({ - timeoutMs: 456, - llm: { - completeJson: async (_messages, opts) => { - seen = opts; - return { - value: { relation: "follow_up", confidence: 0.9, reason: "same task" }, - servedBy: "fake/llm", - } as never; - }, - } as LlmClient, - }); - - await c.classify({ - prevUserText: "investigate retrieval latency", - prevAssistantText: "I found several possible causes.", - newUserText: "could the queue contribute to this behavior?", - signal: controller.signal, - }); - - expect(seen?.signal).toBe(controller.signal); - expect(seen?.timeoutMs).toBe(456); - }); - it("falls back to heuristic when LLM throws", async () => { const llm: Partial = { completeJson: async () => { diff --git a/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts b/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts deleted file mode 100644 index 1380b9a47..000000000 --- a/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { - createForegroundResources, - prioritizeEmbedder, -} from "../../../core/util/foreground-resources.js"; -import { fakeEmbedder } from "../../helpers/fake-embedder.js"; - -describe("foreground resources", () => { - it("admits a queued foreground embedding before queued background work", async () => { - const resources = createForegroundResources({ embeddingConcurrency: 1 }); - const first = await resources.acquireEmbedding("background"); - const order: string[] = []; - - const background = resources.acquireEmbedding("background").then((release) => { - order.push("background"); - release(); - }); - const foreground = resources.acquireEmbedding("foreground").then((release) => { - order.push("foreground"); - release(); - }); - - first(); - await Promise.all([foreground, background]); - - expect(order).toEqual(["foreground", "background"]); - }); - - it("lets background work progress after a bounded foreground burst", async () => { - const resources = createForegroundResources({ - embeddingConcurrency: 1, - maxForegroundBurst: 2, - }); - const first = await resources.acquireEmbedding("foreground"); - const order: string[] = []; - - const background = resources.acquireEmbedding("background").then((release) => { - order.push("background"); - release(); - }); - const foreground1 = resources.acquireEmbedding("foreground").then((release) => { - order.push("foreground-1"); - release(); - }); - const foreground2 = resources.acquireEmbedding("foreground").then((release) => { - order.push("foreground-2"); - release(); - }); - - first(); - await Promise.all([background, foreground1, foreground2]); - - expect(order).toEqual(["foreground-1", "background", "foreground-2"]); - }); - - it("does not start background work while a foreground turn is active", async () => { - const resources = createForegroundResources(); - const leaveForeground = resources.enterForeground(); - let started = false; - - const waiting = resources.waitForBackground().then(() => { - started = true; - }); - await Promise.resolve(); - expect(started).toBe(false); - - leaveForeground(); - await waiting; - expect(started).toBe(true); - }); - - it("removes an aborted embedding waiter without consuming capacity", async () => { - const resources = createForegroundResources({ embeddingConcurrency: 1 }); - const first = await resources.acquireEmbedding("background"); - const controller = new AbortController(); - const waiting = resources.acquireEmbedding("foreground", controller.signal); - - controller.abort(); - await expect(waiting).rejects.toMatchObject({ name: "AbortError" }); - first(); - - const release = await resources.acquireEmbedding("background"); - release(); - }); - - it("chunks background embedding batches and yields between chunks", async () => { - const resources = createForegroundResources({ embeddingConcurrency: 1 }); - const base = fakeEmbedder({ dimensions: 4 }); - const batchSizes: number[] = []; - const inner = { - ...base, - async embedMany(...args: Parameters) { - batchSizes.push(args[0].length); - return base.embedMany(...args); - }, - }; - const background = prioritizeEmbedder(inner, resources, "background", 2)!; - - await background.embedMany(["a", "b", "c", "d", "e"]); - - expect(batchSizes).toEqual([2, 2, 1]); - }); - - it("aborts queued and in-flight provider work during shutdown", async () => { - const resources = createForegroundResources({ embeddingConcurrency: 1 }); - const base = fakeEmbedder({ dimensions: 4 }); - let providerSignal: AbortSignal | undefined; - const inner = { - ...base, - async embedOne( - _input: Parameters[0], - options?: Parameters[1], - ) { - providerSignal = options?.signal; - return await new Promise((_resolve, reject) => { - if (options?.signal?.aborted) { - reject(options.signal.reason); - return; - } - options?.signal?.addEventListener( - "abort", - () => reject(options.signal?.reason), - { once: true }, - ); - }); - }, - }; - const background = prioritizeEmbedder(inner, resources, "background")!; - const pending = background.embedOne("slow background work"); - await Promise.resolve(); - - resources.shutdown("test shutdown"); - - await expect(pending).rejects.toMatchObject({ name: "AbortError" }); - expect(providerSignal?.aborted).toBe(true); - }); -}); diff --git a/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts b/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts deleted file mode 100644 index 8d6b8d5a2..000000000 --- a/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; - -import { createRequestDeadline } from "../../../core/util/request-deadline.js"; - -afterEach(() => { - vi.useRealTimers(); -}); - -describe("createRequestDeadline", () => { - it("aborts at the absolute deadline and reports no remaining budget", async () => { - vi.useFakeTimers(); - vi.setSystemTime(1_000); - - const deadline = createRequestDeadline(1_250); - expect(deadline.remainingMs()).toBe(250); - expect(deadline.signal.aborted).toBe(false); - - await vi.advanceTimersByTimeAsync(250); - - expect(deadline.signal.aborted).toBe(true); - expect(deadline.remainingMs()).toBe(0); - deadline.dispose(); - }); - - it("treats an already-expired deadline as immediately aborted", () => { - vi.useFakeTimers(); - vi.setSystemTime(2_000); - - const deadline = createRequestDeadline(1_999); - - expect(deadline.signal.aborted).toBe(true); - expect(deadline.remainingMs()).toBe(0); - deadline.dispose(); - }); -}); diff --git a/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts b/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts deleted file mode 100644 index 84a4c8ea4..000000000 --- a/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { - clearRetryCooldowns, - getRetryCooldown, - parseRetryAfterMs, - planRetry, - recordRetryCooldown, - retryCooldownKey, -} from "../../../core/util/retry-after.js"; - -describe("parseRetryAfterMs", () => { - it("parses delay-seconds", () => { - expect(parseRetryAfterMs("3", 1_000)).toBe(3_000); - expect(parseRetryAfterMs("0", 1_000)).toBe(0); - }); - - it("parses an HTTP-date relative to the supplied clock", () => { - const now = Date.parse("2026-08-04T00:00:00.000Z"); - expect(parseRetryAfterMs("Tue, 04 Aug 2026 00:00:05 GMT", now)).toBe(5_000); - }); - - it("clamps past HTTP-dates and rejects malformed values", () => { - const now = Date.parse("2026-08-04T00:00:00.000Z"); - expect(parseRetryAfterMs("Mon, 03 Aug 2026 23:59:59 GMT", now)).toBe(0); - expect(parseRetryAfterMs("1.5", now)).toBeNull(); - expect(parseRetryAfterMs("9007199254740991", now)).toBeNull(); - expect(parseRetryAfterMs("later", now)).toBeNull(); - expect(parseRetryAfterMs(null, now)).toBeNull(); - }); - - it("defers instead of retrying before a long provider Retry-After", () => { - expect(planRetry({ - attempt: 1, - baseMs: 200, - jitterMaxMs: 0, - retryAfterMs: 120_000, - maxInlineDelayMs: 30_000, - nowMs: 1_000, - })).toEqual({ - action: "defer", - backoffMs: 200, - delayMs: 120_000, - reason: "retry_after_too_long", - retryAfterMs: 120_000, - retryAt: 121_000, - source: "retry_after", - }); - }); - - it("defers when an otherwise short retry cannot fit the request deadline", () => { - expect(planRetry({ - attempt: 1, - baseMs: 200, - jitterMaxMs: 0, - retryAfterMs: 2_000, - deadlineAt: 2_500, - nowMs: 1_000, - })).toMatchObject({ - action: "defer", - reason: "deadline_insufficient", - retryAt: 3_000, - }); - }); - - it("keeps provider cooldowns monotonic and expires them at retryAt", () => { - clearRetryCooldowns(); - recordRetryCooldown("llm:test", { - retryAfterMs: 2_000, - retryAt: 3_000, - status: 429, - }); - recordRetryCooldown("llm:test", { - retryAfterMs: 500, - retryAt: 1_500, - status: 503, - }); - expect(getRetryCooldown("llm:test", 2_999)).toMatchObject({ - retryAt: 3_000, - status: 429, - }); - expect(getRetryCooldown("llm:test", 3_000)).toBeNull(); - clearRetryCooldowns(); - }); - - it("scopes provider cooldowns by endpoint and model", () => { - expect(retryCooldownKey("llm", "openai_compatible", "https://x", "model-a")) - .not.toBe(retryCooldownKey("llm", "openai_compatible", "https://x", "model-b")); - }); -}); diff --git a/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts b/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts deleted file mode 100644 index 2a5308ad2..000000000 --- a/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { createSemaphore } from "../../../core/util/semaphore.js"; - -describe("semaphore", () => { - it("removes an aborted waiter so shutdown cannot hang behind active work", async () => { - const semaphore = createSemaphore(1); - const release = await semaphore.acquire(); - const controller = new AbortController(); - const waiting = semaphore.acquire(controller.signal); - - controller.abort(); - await expect(waiting).rejects.toMatchObject({ name: "AbortError" }); - release(); - - const next = await semaphore.acquire(); - next(); - }); -}); From 0ff0a72d2e1794301b59e7013c9293883a589813 Mon Sep 17 00:00:00 2001 From: CovD <2643822566@qq.com> Date: Wed, 5 Aug 2026 00:23:08 +0800 Subject: [PATCH 05/34] fix(plugin): bound idle archive batches --- .../core/skill/ALGORITHMS.md | 4 +- .../core/skill/subscriber.ts | 15 +++++- .../tests/unit/skill/subscriber.test.ts | 47 +++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/apps/memos-local-plugin/core/skill/ALGORITHMS.md b/apps/memos-local-plugin/core/skill/ALGORITHMS.md index 94daaf7ab..205c3e7b2 100644 --- a/apps/memos-local-plugin/core/skill/ALGORITHMS.md +++ b/apps/memos-local-plugin/core/skill/ALGORITHMS.md @@ -253,7 +253,9 @@ the next lifecycle tick. `lastUsedAt` is updated by the existing recorded-use path. A never-used skill falls back to `createdAt`; unrelated metadata updates therefore do not reset its idle clock. The scan runs through the orchestrator's normal -flush lifecycle and does not introduce a separate timer. +flush lifecycle and does not introduce a separate timer. Each tick processes +at most ten 500-row batches; any remaining backlog is deferred to a later tick +so a large archive queue cannot monopolize the event loop. --- diff --git a/apps/memos-local-plugin/core/skill/subscriber.ts b/apps/memos-local-plugin/core/skill/subscriber.ts index af1b0a57c..9dd861a73 100644 --- a/apps/memos-local-plugin/core/skill/subscriber.ts +++ b/apps/memos-local-plugin/core/skill/subscriber.ts @@ -37,6 +37,8 @@ import type { SkillId } from "../types.js"; import { now as nowMs } from "../time.js"; import { IDLE_ARCHIVE_BATCH_LIMIT } from "../storage/repos/skills.js"; +const IDLE_ARCHIVE_MAX_BATCHES_PER_TICK = 10; + export interface SkillSubscriberDeps extends Omit { log?: Logger; @@ -230,17 +232,21 @@ export function attachSkillSubscriber( } const cutoff = at - deps.config.idleArchiveMs; - while (true) { + let batchesProcessed = 0; + let archivedTotal = 0; + while (batchesProcessed < IDLE_ARCHIVE_MAX_BATCHES_PER_TICK) { const archiveCandidates = deps.repos.skills.listIdleArchiveCandidates({ minEtaForRetrieval: deps.config.minEtaForRetrieval, cutoff, limit: IDLE_ARCHIVE_BATCH_LIMIT, }); + batchesProcessed += 1; let archivedThisBatch = 0; for (const s of archiveCandidates) { if (!shouldArchiveIdle(s, deps.config.idleArchiveMs, deps.config, at)) continue; deps.repos.skills.setStatus(s.id, "archived", at); archivedThisBatch += 1; + archivedTotal += 1; log.info("skill.idle_archived", { skillId: s.id, name: s.name, @@ -266,6 +272,13 @@ export function attachSkillSubscriber( break; } if (archiveCandidates.length < IDLE_ARCHIVE_BATCH_LIMIT) break; + if (batchesProcessed === IDLE_ARCHIVE_MAX_BATCHES_PER_TICK) { + log.warn("skill.idle_archive_batch_limit_reached", { + batchCount: batchesProcessed, + archivedCount: archivedTotal, + batchSize: IDLE_ARCHIVE_BATCH_LIMIT, + }); + } } } diff --git a/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts b/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts index 88ec1eafb..a8c35468a 100644 --- a/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts @@ -250,4 +250,51 @@ describe("skill/subscriber", () => { expect(h.repos.skills.count({ status: "active" })).toBe(0); sub.dispose(); }); + + it("caps idle archival at ten batches per lifecycle tick", async () => { + handle = makeTmpDb(); + const h = handle; + for (let i = 0; i < 5_001; i++) { + seedSkill(h, { + id: `sk_backlog_${i}` as never, + name: `backlog_skill_${i}`, + status: "active", + eta: 0.05, + createdAt: 1 as never, + updatedAt: (i + 1) as never, + lastUsedAt: 1 as never, + }); + } + const log = rootLogger.child({ channel: "core.skill.subscriber" }); + const infoSpy = vi.spyOn(log, "info").mockImplementation(() => undefined); + const warnSpy = vi.spyOn(log, "warn").mockImplementation(() => undefined); + const sub = attachSkillSubscriber({ + l2Bus: createL2EventBus(), + rewardBus: createRewardEventBus(), + bus: createSkillEventBus(), + repos: h.repos, + embedder: null, + llm: null, + log, + config: makeSkillConfig({ minEtaForRetrieval: 0.1, idleArchiveMs: 1_000 }), + }); + + await sub.lifecycleTick(); + + expect(h.repos.skills.count({ status: "archived" })).toBe(5_000); + expect(h.repos.skills.count({ status: "active" })).toBe(1); + expect(warnSpy).toHaveBeenCalledWith("skill.idle_archive_batch_limit_reached", { + batchCount: 10, + archivedCount: 5_000, + batchSize: 500, + }); + + await sub.lifecycleTick(); + + expect(h.repos.skills.count({ status: "archived" })).toBe(5_001); + expect(h.repos.skills.count({ status: "active" })).toBe(0); + sub.dispose(); + infoSpy.mockRestore(); + warnSpy.mockRestore(); + }); }); From 211dd1655bbb8d2c9d5e0a9b5e21cd82a37563da Mon Sep 17 00:00:00 2001 From: jiachengzhen Date: Thu, 6 Aug 2026 22:55:12 +0800 Subject: [PATCH 06/34] fix(plugin): harden Hermes pgrep pattern --- .../bridge/hermes-process.ts | 40 +++++++++-------- .../tests/unit/bridge/hermes-process.test.ts | 44 +++++++++++-------- 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/apps/memos-local-plugin/bridge/hermes-process.ts b/apps/memos-local-plugin/bridge/hermes-process.ts index b550a1c2a..feaf0b818 100644 --- a/apps/memos-local-plugin/bridge/hermes-process.ts +++ b/apps/memos-local-plugin/bridge/hermes-process.ts @@ -18,25 +18,16 @@ * therefore misses any invocation with a global flag (`--skills`, * `-m`, `--provider`, …) between them. * - * The current pattern is `hermes(\s+\S+)*\s+chat\b`: + * The command grammar is `hermes ()* chat (|$)`: * - * • `hermes` — the binary basename. - * • `(\s+\S+)*` — any complete argv-style tokens between the - * binary and the subcommand. - * • `\s+chat\b` — a standalone `chat` token, so it does *not* - * match `chatter`, `chat-server`, `--chat-log`, or a flag value - * like `--profile=chat`. + * • `hermes` — the binary basename. + * • `()*` — complete argv-style tokens before the subcommand. + * • `chat` — a complete token, not `chatter` or `chat-server`. * - * `pgrep -f` on Linux uses glibc's ERE engine, which supports - * `\s`/`\b` as GNU extensions. ⚠️ The pattern MUST stay within POSIX - * ERE — in particular it must NOT use `(?:…)` non-capturing groups, - * which are PCRE-only: glibc ERE rejects the whole pattern with - * "Invalid preceding regular expression", `pgrep` exits 2, and - * `isHermesChatRunning()` silently reports `false`, leaving the - * viewer stuck on `"disconnected"`. A plain capturing group `(…)` - * is valid in both ERE and JavaScript's `RegExp`, so - * `matchesHermesChatCommandLine()` can still proxy the pattern for - * unit tests without a real Hermes binary or a fork of pgrep in CI. + * `pgrep -f` on Linux uses glibc's POSIX ERE engine, so its pattern uses + * POSIX character classes and capturing groups only. JavaScript does not + * implement POSIX character classes, so the test helper builds the same + * grammar with `\s` and `\S` tokens instead. */ // eslint-disable-next-line @typescript-eslint/no-require-imports import * as childProcess from "node:child_process"; @@ -48,7 +39,18 @@ import * as childProcess from "node:child_process"; * string we hand to `pgrep` and confirm we have not silently regressed * back to a literal substring match. */ -export const HERMES_CHAT_PROCESS_PATTERN = "hermes(\\s+\\S+)*\\s+chat\\b"; +function buildHermesChatPattern(space: string, nonSpace: string): string { + return `hermes(${space}+${nonSpace}+)*${space}+chat(${space}|$)`; +} + +export const HERMES_CHAT_PROCESS_PATTERN = buildHermesChatPattern( + "[[:space:]]", + "[^[:space:]]", +); + +const HERMES_CHAT_JS_PATTERN = new RegExp( + buildHermesChatPattern("\\s", "\\S"), +); /** * JS-side equivalent of `pgrep -f HERMES_CHAT_PROCESS_PATTERN`. @@ -59,7 +61,7 @@ export const HERMES_CHAT_PROCESS_PATTERN = "hermes(\\s+\\S+)*\\s+chat\\b"; * `/proc//cmdline`-style command-line string. */ export function matchesHermesChatCommandLine(commandLine: string): boolean { - return new RegExp(HERMES_CHAT_PROCESS_PATTERN).test(commandLine); + return HERMES_CHAT_JS_PATTERN.test(commandLine); } /** diff --git a/apps/memos-local-plugin/tests/unit/bridge/hermes-process.test.ts b/apps/memos-local-plugin/tests/unit/bridge/hermes-process.test.ts index ddd397d57..5b5ed7c33 100644 --- a/apps/memos-local-plugin/tests/unit/bridge/hermes-process.test.ts +++ b/apps/memos-local-plugin/tests/unit/bridge/hermes-process.test.ts @@ -7,14 +7,12 @@ * subcommand (`hermes --skills memory-routing chat`) was silently * missed and the viewer was stuck on `"disconnected"`. * - * The pattern under test is `hermes(\s+\S+)*\s+chat\b` — these cases - * lock in the exact shape of the fix. It must stay valid POSIX ERE - * (no `(?:…)` groups): glibc ERE rejects non-capturing groups, so a - * PCRE-ism in the pattern makes every `pgrep -f` call fail with a - * regex error and `isHermesChatRunning()` return `false` forever. + * The pgrep pattern uses POSIX character classes while the JS helper + * uses equivalent `\s` / `\S` tokens. These cases lock in both the + * shared command grammar and the exact wire format passed to pgrep. */ -import { describe, expect, it, vi } from "vitest"; import { spawnSync } from "node:child_process"; +import { describe, expect, it, vi } from "vitest"; import { HERMES_CHAT_PROCESS_PATTERN, @@ -27,21 +25,23 @@ describe("HERMES_CHAT_PROCESS_PATTERN", () => { // If this string ever changes, audit `bridge.cts` callers and the // issue description before adjusting — the constant is the only // surface that fixes the substring-detection bug. - expect(HERMES_CHAT_PROCESS_PATTERN).toBe("hermes(\\s+\\S+)*\\s+chat\\b"); + expect(HERMES_CHAT_PROCESS_PATTERN).toBe( + "hermes([[:space:]]+[^[:space:]]+)*[[:space:]]+chat([[:space:]]|$)", + ); }); - it("compiles under glibc POSIX ERE — pgrep must not exit 2 (regex error)", () => { - // Regression for the `(?:…)` non-capturing group: JS RegExp accepts - // it, but glibc ERE (what `pgrep -f` uses on Linux) rejects the - // whole pattern with "Invalid preceding regular expression". Run the - // real binary so a PCRE-ism can never silently sneak back in. - // exit 0 = match, 1 = no match (both fine), 2 = regex syntax error. - const result = spawnSync("pgrep", ["-f", HERMES_CHAT_PROCESS_PATTERN], { - encoding: "utf8", - timeout: 2000, - }); - expect(result.status).not.toBe(2); - }); + it.skipIf(process.platform !== "linux")( + "compiles under the glibc ERE engine used by pgrep", + () => { + const result = spawnSync("pgrep", ["-f", HERMES_CHAT_PROCESS_PATTERN], { + encoding: "utf8", + timeout: 2000, + }); + + expect(result.error).toBeUndefined(); + expect([0, 1]).toContain(result.status); + }, + ); }); describe("matchesHermesChatCommandLine", () => { @@ -95,6 +95,12 @@ describe("matchesHermesChatCommandLine", () => { ).toBe(false); }); + it("does not match `hermes chat-server` (chat must be a complete token)", () => { + expect( + matchesHermesChatCommandLine("/usr/local/bin/hermes chat-server"), + ).toBe(false); + }); + it("does not match `hermes --chat-log=... status` (chat must be the subcommand token)", () => { expect( matchesHermesChatCommandLine( From d985d076129ff5656f6c0094537abf719078adf5 Mon Sep 17 00:00:00 2001 From: jiachengzhen Date: Thu, 6 Aug 2026 22:58:47 +0800 Subject: [PATCH 07/34] refactor(plugin): avoid JS regex captures --- apps/memos-local-plugin/bridge/hermes-process.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/memos-local-plugin/bridge/hermes-process.ts b/apps/memos-local-plugin/bridge/hermes-process.ts index feaf0b818..e9ecb84f5 100644 --- a/apps/memos-local-plugin/bridge/hermes-process.ts +++ b/apps/memos-local-plugin/bridge/hermes-process.ts @@ -39,8 +39,12 @@ import * as childProcess from "node:child_process"; * string we hand to `pgrep` and confirm we have not silently regressed * back to a literal substring match. */ -function buildHermesChatPattern(space: string, nonSpace: string): string { - return `hermes(${space}+${nonSpace}+)*${space}+chat(${space}|$)`; +function buildHermesChatPattern( + space: string, + nonSpace: string, + groupStart = "(", +): string { + return `hermes${groupStart}${space}+${nonSpace}+)*${space}+chat${groupStart}${space}|$)`; } export const HERMES_CHAT_PROCESS_PATTERN = buildHermesChatPattern( @@ -49,7 +53,7 @@ export const HERMES_CHAT_PROCESS_PATTERN = buildHermesChatPattern( ); const HERMES_CHAT_JS_PATTERN = new RegExp( - buildHermesChatPattern("\\s", "\\S"), + buildHermesChatPattern("\\s", "\\S", "(?:"), ); /** From 02aa0d8b1893f3a9cca3f2518f351a0993a8b38b Mon Sep 17 00:00:00 2001 From: jiachengzhen Date: Thu, 6 Aug 2026 23:02:19 +0800 Subject: [PATCH 08/34] refactor(plugin): clarify Hermes regex variants --- .../bridge/hermes-process.ts | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/apps/memos-local-plugin/bridge/hermes-process.ts b/apps/memos-local-plugin/bridge/hermes-process.ts index e9ecb84f5..b796e2d90 100644 --- a/apps/memos-local-plugin/bridge/hermes-process.ts +++ b/apps/memos-local-plugin/bridge/hermes-process.ts @@ -26,8 +26,8 @@ * * `pgrep -f` on Linux uses glibc's POSIX ERE engine, so its pattern uses * POSIX character classes and capturing groups only. JavaScript does not - * implement POSIX character classes, so the test helper builds the same - * grammar with `\s` and `\S` tokens instead. + * implement POSIX character classes, so the test helper declares the same + * grammar with `\s`, `\S`, and non-capturing groups instead. */ // eslint-disable-next-line @typescript-eslint/no-require-imports import * as childProcess from "node:child_process"; @@ -39,22 +39,12 @@ import * as childProcess from "node:child_process"; * string we hand to `pgrep` and confirm we have not silently regressed * back to a literal substring match. */ -function buildHermesChatPattern( - space: string, - nonSpace: string, - groupStart = "(", -): string { - return `hermes${groupStart}${space}+${nonSpace}+)*${space}+chat${groupStart}${space}|$)`; -} - -export const HERMES_CHAT_PROCESS_PATTERN = buildHermesChatPattern( - "[[:space:]]", - "[^[:space:]]", -); +export const HERMES_CHAT_PROCESS_PATTERN = + "hermes([[:space:]]+[^[:space:]]+)*[[:space:]]+chat([[:space:]]|$)"; -const HERMES_CHAT_JS_PATTERN = new RegExp( - buildHermesChatPattern("\\s", "\\S", "(?:"), -); +// Keep this semantically aligned with HERMES_CHAT_PROCESS_PATTERN. POSIX ERE +// has no non-capturing groups, while JavaScript can avoid unused captures. +const HERMES_CHAT_JS_PATTERN = /hermes(?:\s+\S+)*\s+chat(?:\s|$)/; /** * JS-side equivalent of `pgrep -f HERMES_CHAT_PROCESS_PATTERN`. From 11b6a5415649bab342780ff4e79d3a058764d6a9 Mon Sep 17 00:00:00 2001 From: jiachengzhen Date: Thu, 6 Aug 2026 23:29:26 +0800 Subject: [PATCH 09/34] chore(plugin): sync idle archive PR with dev-v2.0.29 --- .../hermes/memos_provider/__init__.py | 147 +++++++++- .../hermes/memos_provider/bridge_client.py | 137 ++++++--- apps/memos-local-plugin/agent-contract/dto.ts | 5 + apps/memos-local-plugin/bridge/methods.ts | 9 + .../core/embedding/embedder.ts | 18 +- .../core/embedding/fetcher.ts | 129 ++++++++- .../core/embedding/index.ts | 1 + .../core/embedding/providers/cohere.ts | 4 +- .../core/embedding/providers/gemini.ts | 4 +- .../core/embedding/providers/mistral.ts | 4 +- .../core/embedding/providers/openai.ts | 4 +- .../core/embedding/providers/voyage.ts | 4 +- .../core/embedding/retry-worker.ts | 31 +- .../core/embedding/types.ts | 20 +- apps/memos-local-plugin/core/index.ts | 1 + apps/memos-local-plugin/core/llm/client.ts | 11 + apps/memos-local-plugin/core/llm/fetcher.ts | 137 ++++++++- .../core/llm/providers/anthropic.ts | 4 +- .../core/llm/providers/bedrock.ts | 4 +- .../core/llm/providers/gemini.ts | 4 +- .../core/llm/providers/openai.ts | 4 +- apps/memos-local-plugin/core/llm/types.ts | 9 +- apps/memos-local-plugin/core/pipeline/deps.ts | 29 +- .../core/pipeline/memory-core.ts | 88 +++--- .../core/pipeline/orchestrator.ts | 153 ++++++++-- .../core/retrieval/llm-filter.ts | 8 + .../core/retrieval/retrieve.ts | 53 +++- .../core/retrieval/types.ts | 6 +- .../core/session/intent-classifier.ts | 8 +- .../core/session/manager.ts | 3 + .../core/session/relation-classifier.ts | 24 +- apps/memos-local-plugin/core/session/types.ts | 2 + .../core/util/foreground-resources.ts | 274 ++++++++++++++++++ .../core/util/rate-limited-llm.ts | 31 +- .../core/util/request-deadline.ts | 37 +++ .../core/util/retry-after.ts | 200 +++++++++++++ .../memos-local-plugin/core/util/semaphore.ts | 43 ++- .../tests/python/test_bridge_client.py | 165 ++++++++++- .../python/test_hermes_provider_pipeline.py | 69 +++++ .../tests/unit/bridge/methods.test.ts | 11 + .../tests/unit/embedding/embedder.test.ts | 59 ++++ .../tests/unit/embedding/fetcher.test.ts | 82 ++++++ .../tests/unit/embedding/retry-worker.test.ts | 60 ++++ .../tests/unit/llm/client.test.ts | 39 +++ .../tests/unit/llm/fetcher.test.ts | 201 ++++++++++++- .../tests/unit/pipeline/memory-core.test.ts | 47 +++ .../tests/unit/pipeline/orchestrator.test.ts | 33 +++ .../unit/session/intent-classifier.test.ts | 19 ++ .../unit/session/relation-classifier.test.ts | 27 ++ .../unit/util/foreground-resources.test.ts | 138 +++++++++ .../tests/unit/util/request-deadline.test.ts | 35 +++ .../tests/unit/util/retry-after.test.ts | 90 ++++++ .../tests/unit/util/semaphore.test.ts | 19 ++ 53 files changed, 2547 insertions(+), 197 deletions(-) create mode 100644 apps/memos-local-plugin/core/util/foreground-resources.ts create mode 100644 apps/memos-local-plugin/core/util/request-deadline.ts create mode 100644 apps/memos-local-plugin/core/util/retry-after.ts create mode 100644 apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts create mode 100644 apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts create mode 100644 apps/memos-local-plugin/tests/unit/util/retry-after.test.ts create mode 100644 apps/memos-local-plugin/tests/unit/util/semaphore.test.ts diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index 6eeaa000e..79b88cbf4 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -201,6 +201,47 @@ def _long_rpc_timeout_default() -> float: _LONG_RPC_TIMEOUT = _long_rpc_timeout_default() + +def _prefetch_rpc_timeout_default() -> float: + """Resolve the latency budget for Hermes' foreground memory lookup. + + Hermes places its own short deadline around ``prefetch``. Reusing the + long capture timeout here lets the bridge continue work after the host + has already moved on. Keep a separate, configurable ceiling below the + host's default and forward the corresponding absolute deadline to core. + """ + raw = os.environ.get("MEMOS_HERMES_PREFETCH_RPC_TIMEOUT", "") + try: + value = float(raw) + except (TypeError, ValueError): + return 6.0 + if not value > 0: + return 6.0 + # Hermes currently abandons external providers after 8 seconds. Keep at + # least one second for Python thread scheduling and response assembly even + # when an operator overrides the default. + return min(value, 7.0) + + +_PREFETCH_RPC_TIMEOUT = _prefetch_rpc_timeout_default() +_PREFETCH_RESPONSE_RESERVE_SECONDS = 0.25 + + +def _remaining_rpc_timeout( + deadline_monotonic: float | None, + requested_timeout: float | None, +) -> float | None: + """Bound one blocking bridge step by a shared end-to-end deadline.""" + if deadline_monotonic is None: + return requested_timeout + remaining = deadline_monotonic - time.monotonic() + if remaining <= 0: + raise BridgeError("timeout", "foreground prefetch deadline exceeded") + if requested_timeout is None: + return remaining + return min(requested_timeout, remaining) + + _HERMES_INTERNAL_REVIEW_PREFIXES = ( "review the conversation above and consider saving to memory if appropriate.", "review the conversation above and update the skill library.", @@ -1022,8 +1063,19 @@ def prefetch(self, query: str, *, session_id: str = "") -> str: # type: ignore[ cached result immediately. Otherwise synchronously run ``turn.start`` against the bridge (small overhead). """ + deadline_monotonic = time.monotonic() + _PREFETCH_RPC_TIMEOUT + started_at_ms = int(time.time() * 1000) + core_budget_seconds = max( + 0.05, + _PREFETCH_RPC_TIMEOUT - _PREFETCH_RESPONSE_RESERVE_SECONDS, + ) + deadline_at_ms = started_at_ms + int(core_budget_seconds * 1000) if self._prefetch_thread and self._prefetch_thread.is_alive(): - self._prefetch_thread.join(timeout=5.0) + try: + join_timeout = _remaining_rpc_timeout(deadline_monotonic, 5.0) + except BridgeError: + return "" + self._prefetch_thread.join(timeout=join_timeout) with self._prefetch_lock: cached = self._prefetch_result self._prefetch_result = "" @@ -1033,10 +1085,25 @@ def prefetch(self, query: str, *, session_id: str = "") -> str: # type: ignore[ suppress_injection = _is_explicit_delegation_request(query) if cached: return "" if suppress_injection else cached - if not self._ensure_bridge(session_id or self._session_id, timeout=10.0): + try: + ensure_timeout = _remaining_rpc_timeout( + deadline_monotonic, + _PREFETCH_RPC_TIMEOUT, + ) + except BridgeError: + return "" + if not self._ensure_bridge( + session_id or self._session_id, + timeout=min(10.0, ensure_timeout or _PREFETCH_RPC_TIMEOUT), + ): return "" try: - context = self._turn_start(query, session_id=session_id) + context = self._turn_start( + query, + session_id=session_id, + deadline_monotonic=deadline_monotonic, + deadline_at_ms=deadline_at_ms, + ) if suppress_injection: # Do not let remembered "do it directly" skills override an # explicit user request to dispatch work to a subagent. @@ -1941,6 +2008,7 @@ def _bridge_request( *, timeout: float | None = None, ensure_session: bool = True, + deadline_monotonic: float | None = None, ) -> dict[str, Any]: bridge = self._bridge if bridge is None: @@ -1958,10 +2026,23 @@ def _bridge_request( bridge.generation, self._session_id, ) - self._open_session(self._session_id, timeout=30.0) - if timeout is None: + session_ceiling = ( + timeout + if deadline_monotonic is not None and timeout is not None + else 30.0 + ) + session_timeout = _remaining_rpc_timeout( + deadline_monotonic, + session_ceiling, + ) + self._open_session( + self._session_id, + timeout=session_timeout or 30.0, + ) + request_timeout = _remaining_rpc_timeout(deadline_monotonic, timeout) + if request_timeout is None: return bridge.request(method, params) - return bridge.request(method, params, timeout=timeout) + return bridge.request(method, params, timeout=request_timeout) def _open_session(self, session_id: str = "", *, timeout: float = 30.0) -> None: bridge = self._bridge @@ -1997,6 +2078,7 @@ def _bridge_request_with_retry( params: Any, *, timeout: float | None = None, + deadline_monotonic: float | None = None, ) -> dict[str, Any]: """Read-path helper: reconnect + retry once on ``transport_closed``. @@ -2012,7 +2094,12 @@ def _bridge_request_with_retry( """ assert self._bridge is not None try: - return self._bridge_request(method, params, timeout=timeout) + return self._bridge_request( + method, + params, + timeout=timeout, + deadline_monotonic=deadline_monotonic, + ) except BridgeError as err: if not self._is_transport_closed(err): raise @@ -2021,9 +2108,24 @@ def _bridge_request_with_retry( method, err, ) - self._reconnect_bridge(self._session_id, timeout=30.0) + reconnect_ceiling = ( + timeout if deadline_monotonic is not None and timeout is not None else 30.0 + ) + reconnect_timeout = _remaining_rpc_timeout( + deadline_monotonic, + reconnect_ceiling, + ) + self._reconnect_bridge( + self._session_id, + timeout=reconnect_timeout or 30.0, + ) assert self._bridge is not None - return self._bridge_request(method, params, timeout=timeout) + return self._bridge_request( + method, + params, + timeout=timeout, + deadline_monotonic=deadline_monotonic, + ) def _is_transport_closed(self, err: Exception) -> bool: if isinstance(err, BridgeError) and err.code == "transport_closed": @@ -2233,7 +2335,14 @@ def _run() -> None: ) self._bridge_keepalive_thread.start() - def _turn_start(self, query: str, *, session_id: str = "") -> str: + def _turn_start( + self, + query: str, + *, + session_id: str = "", + deadline_monotonic: float | None = None, + deadline_at_ms: int | None = None, + ) -> str: assert self._bridge is not None host_runtime = self._host_runtime_context() with self._state_lock: @@ -2251,20 +2360,34 @@ def _turn_start(self, query: str, *, session_id: str = "") -> str: "visibleContextStartTs": visible_context_start_ts, } ) + now_ms = int(time.time() * 1000) + if deadline_monotonic is None: + deadline_monotonic = time.monotonic() + _PREFETCH_RPC_TIMEOUT + if deadline_at_ms is None: + core_budget_seconds = max( + 0.05, + _PREFETCH_RPC_TIMEOUT - _PREFETCH_RESPONSE_RESERVE_SECONDS, + ) + deadline_at_ms = now_ms + int(core_budget_seconds * 1000) payload: dict[str, Any] = { "agent": "hermes", "namespace": self._runtime_namespace(), "sessionId": session_id or self._session_id, "userText": query, "contextHints": context_hints, - "ts": int(time.time() * 1000), + "ts": now_ms, + "deadlineAt": deadline_at_ms, } if turn_key: payload["turnKey"] = turn_key resp = self._bridge_request_with_retry( "turn.start", payload, - timeout=_LONG_RPC_TIMEOUT, + timeout=_remaining_rpc_timeout( + deadline_monotonic, + _PREFETCH_RPC_TIMEOUT, + ), + deadline_monotonic=deadline_monotonic, ) response_query = (resp or {}).get("query") or {} response_session = str(response_query.get("sessionId") or "") diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py index 6863ba1ef..cbb2fa84c 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py @@ -17,6 +17,7 @@ import json import logging import os +import queue import shutil import subprocess import threading @@ -32,6 +33,7 @@ logger = logging.getLogger(__name__) HOST_HANDLER_WAIT_SECONDS = 5.0 +HOST_HANDLER_QUEUE_CAPACITY = 16 # ─── Module-level singleton tracker ───────────────────────────────────── # Each entry maps an ``(agent, no_viewer, runtime_home)`` key to the @@ -152,12 +154,16 @@ def __init__( # Reverse-direction handlers: the bridge can send us a # JSON-RPC request via `serverRequest(...)` (e.g. # `host.llm.complete` for fallback LLM calls). Registered - # methods run on the dedicated reader thread; long-running - # work should spawn its own worker if it needs to. Each - # handler returns a JSON-serialisable value or raises to - # surface a JSON-RPC error back to the bridge. + # methods run on one bounded, daemon worker. Keeping execution + # serial preserves the adapter's previous concurrency contract while + # preventing a slow host LLM call from blocking stdout response + # demultiplexing for every shared provider lease. self._host_handlers: dict[str, Callable[[dict[str, Any]], Any]] = {} self._host_handlers_cv = threading.Condition() + self._host_handler_queue: queue.Queue[tuple[Any, str, dict[str, Any]] | None] = queue.Queue( + maxsize=HOST_HANDLER_QUEUE_CAPACITY + ) + self._host_handler_stop = threading.Event() self._closed = False plugin_root = Path(__file__).resolve().parent.parent.parent.parent @@ -226,6 +232,12 @@ def __init__( env=env, cwd=str(plugin_root), ) + self._host_handler_worker = threading.Thread( + target=self._host_handler_loop, + daemon=True, + name="memos-bridge-host-handler", + ) + self._host_handler_worker.start() self._reader = threading.Thread( target=self._read_loop, daemon=True, @@ -348,7 +360,7 @@ def notify(self, method: str, params: Any = None) -> None: try: self._proc.stdin.write(payload + "\n") self._proc.stdin.flush() - except (BrokenPipeError, OSError): + except (BrokenPipeError, OSError, ValueError): pass def on_event(self, cb: Callable[[dict[str, Any]], None]) -> None: @@ -365,11 +377,9 @@ def register_host_handler( """Register a handler for bridge → adapter (reverse) requests. The Node-side bridge calls these via ``stdio.serverRequest``. - Most-recent registration wins. The handler runs on the reader - thread; if it blocks for a long time it stalls every other - bridge → adapter notification, so handlers that need to do - heavy work (e.g. an LLM call) are still expected to return - within the bridge-side timeout (default 60 s). + Most-recent registration wins. Handlers run serially on a bounded + daemon worker so a long-running host LLM call cannot stall the reader + thread that resolves unrelated foreground JSON-RPC responses. """ with self._host_handlers_cv: self._host_handlers[method] = handler @@ -381,6 +391,7 @@ def close(self) -> None: with self._host_handlers_cv: self._closed = True self._host_handlers_cv.notify_all() + self._stop_host_handler_worker() # Drop self from the module-level singleton tracker (issue #1910) # BEFORE the potentially-slow stdin/SIGTERM/SIGKILL dance. We @@ -435,6 +446,7 @@ def _abort_pending(self, reason: str) -> None: with self._host_handlers_cv: self._closed = True self._host_handlers_cv.notify_all() + self._stop_host_handler_worker() with self._lock: for entry in list(self._pending.values()): entry["error"] = { @@ -477,7 +489,9 @@ def _read_loop(self) -> None: # Reverse-direction request: the bridge is asking the # adapter to do something (e.g. run a fallback LLM call # via `host.llm.complete`). Dispatch to the registered - # handler and write the response back synchronously. + # handler on the bounded worker. The reader must return to + # stdout immediately so a slow host LLM callback cannot + # head-of-line block normal JSON-RPC responses. method = msg.get("method") rpc_id = msg.get("id") if ( @@ -486,33 +500,10 @@ def _read_loop(self) -> None: and "result" not in msg and "error" not in msg ): - handler = self._host_handler_for(method) - if handler is None: - self._send_response( - rpc_id, - error={ - "code": -32601, - "message": f"method not found: {method}", - "data": {"code": "unknown_method"}, - }, - ) - continue params = msg.get("params") or {} if not isinstance(params, dict): params = {} - try: - result = handler(params) - self._send_response(rpc_id, result=result) - except Exception as err: - logger.warning("host handler %s failed: %s", method, err) - self._send_response( - rpc_id, - error={ - "code": -32000, - "message": str(err) or err.__class__.__name__, - "data": {"code": "host_handler_failed"}, - }, - ) + self._dispatch_host_request(rpc_id, method, params) continue except Exception: # Any unexpected exception in the reader loop still needs @@ -527,6 +518,80 @@ def _read_loop(self) -> None: # instead of waiting for each 30 s per-request timeout. self._abort_pending("bridge subprocess exited") + def _dispatch_host_request( + self, + rpc_id: Any, + method: str, + params: dict[str, Any], + ) -> None: + """Queue reverse RPC work without ever blocking the reader thread.""" + if self._closed: + return + try: + self._host_handler_queue.put_nowait((rpc_id, method, params)) + except queue.Full: + logger.warning("host handler queue full; rejecting %s", method) + self._send_response( + rpc_id, + error={ + "code": -32000, + "message": "host handler queue is full", + "data": {"code": "host_handler_busy"}, + }, + ) + + def _host_handler_loop(self) -> None: + """Run reverse RPC handlers serially away from stdout demultiplexing.""" + while True: + request = self._host_handler_queue.get() + try: + if request is None: + return + rpc_id, method, params = request + if self._closed: + continue + handler = self._host_handler_for(method) + if handler is None: + self._send_response( + rpc_id, + error={ + "code": -32601, + "message": f"method not found: {method}", + "data": {"code": "unknown_method"}, + }, + ) + continue + try: + result = handler(params) + self._send_response(rpc_id, result=result) + except Exception as err: + logger.warning("host handler %s failed: %s", method, err) + self._send_response( + rpc_id, + error={ + "code": -32000, + "message": str(err) or err.__class__.__name__, + "data": {"code": "host_handler_failed"}, + }, + ) + finally: + self._host_handler_queue.task_done() + + def _stop_host_handler_worker(self) -> None: + """Discard queued callbacks and ask the daemon worker to exit.""" + if self._host_handler_stop.is_set(): + return + self._host_handler_stop.set() + while True: + try: + self._host_handler_queue.get_nowait() + except queue.Empty: + break + else: + self._host_handler_queue.task_done() + with contextlib.suppress(queue.Full): + self._host_handler_queue.put_nowait(None) + def _host_handler_for( self, method: str, @@ -568,7 +633,7 @@ def _send_response( try: self._proc.stdin.write(json.dumps(payload, ensure_ascii=False) + "\n") self._proc.stdin.flush() - except (BrokenPipeError, OSError): + except (BrokenPipeError, OSError, ValueError): pass def _stderr_loop(self) -> None: diff --git a/apps/memos-local-plugin/agent-contract/dto.ts b/apps/memos-local-plugin/agent-contract/dto.ts index e9ae71101..b76232bee 100644 --- a/apps/memos-local-plugin/agent-contract/dto.ts +++ b/apps/memos-local-plugin/agent-contract/dto.ts @@ -103,6 +103,11 @@ export interface TurnInputDTO { contextHints?: Record; /** Wall-clock when the turn began. */ ts: EpochMs; + /** + * Absolute adapter deadline for foreground work. Every pipeline stage + * shares this budget; it is not reset after relation or intent handling. + */ + deadlineAt?: EpochMs; } export interface TurnResultDTO { diff --git a/apps/memos-local-plugin/bridge/methods.ts b/apps/memos-local-plugin/bridge/methods.ts index 2bc1084a6..1e6cc27a0 100644 --- a/apps/memos-local-plugin/bridge/methods.ts +++ b/apps/memos-local-plugin/bridge/methods.ts @@ -389,6 +389,15 @@ function validateTurnInput(p: Record): void { "turn.start: optional 'turnKey' must be a string", ); } + if ( + p.deadlineAt !== undefined && + (typeof p.deadlineAt !== "number" || !Number.isFinite(p.deadlineAt)) + ) { + throw new MemosError( + "invalid_argument", + "turn.start: optional 'deadlineAt' must be a finite number", + ); + } } function validateTurnResult(p: Record): void { diff --git a/apps/memos-local-plugin/core/embedding/embedder.ts b/apps/memos-local-plugin/core/embedding/embedder.ts index bd4a5fe92..5d035b6ce 100644 --- a/apps/memos-local-plugin/core/embedding/embedder.ts +++ b/apps/memos-local-plugin/core/embedding/embedder.ts @@ -18,6 +18,7 @@ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; import { rootLogger } from "../logger/index.js"; import type { Logger } from "../logger/types.js"; import type { EmbeddingVector } from "../types.js"; +import { extractRetryDiagnostics } from "../util/retry-after.js"; import { LruEmbedCache, NullEmbedCache, @@ -32,6 +33,7 @@ import { MistralEmbeddingProvider } from "./providers/mistral.js"; import { OpenAiEmbeddingProvider } from "./providers/openai.js"; import { VoyageEmbeddingProvider } from "./providers/voyage.js"; import type { + EmbedCallOptions, EmbedInput, EmbedRole, EmbedStats, @@ -87,6 +89,10 @@ export function createEmbedderWithProvider( code?: string; at?: number; durationMs?: number; + retryAfterMs?: number; + retryAt?: number; + retryDecision?: "wait" | "defer" | "stop"; + retryReason?: string; }): void { if (!config.onStatus) return; try { @@ -96,13 +102,17 @@ export function createEmbedderWithProvider( } } - async function embedOne(input: string | EmbedInput): Promise { - const vecs = await embedMany([input]); + async function embedOne( + input: string | EmbedInput, + options?: EmbedCallOptions, + ): Promise { + const vecs = await embedMany([input], options); return vecs[0]!; } async function embedMany( inputs: Array, + options?: EmbedCallOptions, ): Promise { requests += inputs.length; if (inputs.length === 0) return []; @@ -179,6 +189,8 @@ export function createEmbedderWithProvider( const ctx: ProviderCallCtx = { config, log: providerCtxLog, + signal: options?.signal, + deadlineAt: options?.deadlineAt, }; raw = await provider.embed(texts, role, ctx); // Record success but DO NOT clear `lastError` — the viewer @@ -223,6 +235,7 @@ export function createEmbedderWithProvider( message: errMessage, code: err instanceof MemosError ? err.code : undefined, at: errAt, + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); } catch { /* sink errors are non-fatal */ @@ -236,6 +249,7 @@ export function createEmbedderWithProvider( code: err instanceof MemosError ? err.code : undefined, at: errAt, durationMs: errAt - startedAt, + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); throw err instanceof MemosError ? err diff --git a/apps/memos-local-plugin/core/embedding/fetcher.ts b/apps/memos-local-plugin/core/embedding/fetcher.ts index 40524aac7..303dae28e 100644 --- a/apps/memos-local-plugin/core/embedding/fetcher.ts +++ b/apps/memos-local-plugin/core/embedding/fetcher.ts @@ -8,6 +8,15 @@ */ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; +import { + getRetryCooldown, + parseRetryAfterMs, + planRetry, + recordRetryCooldown, + retryCooldownKey, + type RetryPlan, + waitForRetry, +} from "../util/retry-after.js"; import type { EmbeddingProviderName, ProviderLogger } from "./types.js"; export interface HttpPostOpts { @@ -17,6 +26,10 @@ export interface HttpPostOpts { timeoutMs?: number; maxRetries?: number; signal?: AbortSignal; + /** Absolute end-to-end deadline. Unlike timeoutMs, this is not renewed per attempt. */ + deadlineAt?: number; + /** Model/deployment scope; prevents one model cooldown from blocking another. */ + cooldownScope?: string; provider: EmbeddingProviderName; log: ProviderLogger; } @@ -26,11 +39,33 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< const maxRetries = opts.maxRetries ?? 2; let attempt = 0; let lastErr: unknown = null; + const cooldownKey = retryCooldownKey("embedding", opts.provider, opts.url, opts.cooldownScope); while (attempt <= maxRetries) { attempt++; const start = Date.now(); try { + const cooldown = getRetryCooldown(cooldownKey, start); + if (cooldown) { + const details = { + provider: opts.provider, + url: opts.url, + status: cooldown.status, + attempt, + maxRetries, + retryAfterMs: cooldown.retryAfterMs, + retryAt: cooldown.retryAt, + retryDecision: "defer", + retryReason: "cooldown_active", + remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt, start), + }; + opts.log.warn("http.retry_cooldown", details); + throw new MemosError( + ERROR_CODES.EMBEDDING_UNAVAILABLE, + `${opts.provider} is cooling down until ${new Date(cooldown.retryAt).toISOString()}`, + details, + ); + } const signal = mergeSignals(opts.signal, AbortSignal.timeout(timeoutMs)); const resp = await fetch(opts.url, { method: "POST", @@ -46,21 +81,62 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< if (!resp.ok) { const text = await safeText(resp); const transient = resp.status >= 500 || resp.status === 429; + const retryAfterMs = resp.status === 429 || resp.status === 503 + ? parseRetryAfterMs(resp.headers.get("Retry-After")) + : null; + if (retryAfterMs !== null) { + recordRetryCooldown(cooldownKey, { + retryAfterMs, + retryAt: Date.now() + retryAfterMs, + status: resp.status, + }); + } opts.log.warn("http.non_ok", { url: opts.url, status: resp.status, attempt, transient, + retryAfterMs, durationMs: Date.now() - start, }); if (transient && attempt <= maxRetries) { - await backoff(attempt); + const plan = planRetry({ + attempt, + baseMs: 200, + jitterMaxMs: 100, + retryAfterMs, + deadlineAt: opts.deadlineAt, + }); + const retryDetails = retryPlanDetails(plan, opts, maxRetries, resp.status, attempt); + if (plan.action === "defer") { + opts.log.warn("http.retry_deferred", retryDetails); + throw new MemosError( + ERROR_CODES.EMBEDDING_UNAVAILABLE, + `HTTP ${resp.status} from ${opts.provider}; retry deferred until ${new Date(plan.retryAt).toISOString()}`, + retryDetails, + ); + } + opts.log.warn("http.retry_scheduled", retryDetails); + await waitForRetry(plan.delayMs, opts.signal); continue; } throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, `HTTP ${resp.status} from ${opts.provider}`, - { provider: opts.provider, url: opts.url, status: resp.status, body: text }, + { + provider: opts.provider, + url: opts.url, + status: resp.status, + body: text, + ...(retryAfterMs === null + ? {} + : { + retryAfterMs, + retryAt: Date.now() + retryAfterMs, + retryDecision: "stop", + retryReason: "retries_exhausted", + }), + }, ); } @@ -83,7 +159,23 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< durationMs: Date.now() - start, }); if (transient && attempt <= maxRetries) { - await backoff(attempt); + const plan = planRetry({ + attempt, + baseMs: 200, + jitterMaxMs: 100, + deadlineAt: opts.deadlineAt, + }); + const retryDetails = retryPlanDetails(plan, opts, maxRetries, null, attempt); + if (plan.action === "defer") { + opts.log.warn("http.retry_deferred", retryDetails); + throw new MemosError( + ERROR_CODES.EMBEDDING_UNAVAILABLE, + `${opts.provider} retry cannot fit the request deadline`, + retryDetails, + ); + } + opts.log.warn("http.retry_scheduled", retryDetails); + await waitForRetry(plan.delayMs, opts.signal); continue; } throw new MemosError( @@ -123,11 +215,32 @@ function isTransientError(err: unknown): boolean { return false; } -async function backoff(attempt: number): Promise { - const base = 200; - const jitter = Math.floor(Math.random() * 100); - const ms = base * 2 ** (attempt - 1) + jitter; - await new Promise((r) => setTimeout(r, ms)); +function retryPlanDetails( + plan: RetryPlan, + opts: HttpPostOpts, + maxRetries: number, + status: number | null, + attempt: number, +): Record { + return { + provider: opts.provider, + url: opts.url, + status, + attempt, + maxRetries, + backoffMs: plan.backoffMs, + plannedDelayMs: plan.delayMs, + retryAfterMs: plan.retryAfterMs, + retryAt: plan.retryAt, + retrySource: plan.source, + retryDecision: plan.action, + ...(plan.action === "defer" ? { retryReason: plan.reason } : {}), + remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt), + }; +} + +function remainingDeadlineMs(deadlineAt?: number, nowMs: number = Date.now()): number | null { + return deadlineAt === undefined ? null : Math.max(0, deadlineAt - nowMs); } function mergeSignals(a: AbortSignal | undefined, b: AbortSignal): AbortSignal { diff --git a/apps/memos-local-plugin/core/embedding/index.ts b/apps/memos-local-plugin/core/embedding/index.ts index 99faa0048..f6f4b1ed9 100644 --- a/apps/memos-local-plugin/core/embedding/index.ts +++ b/apps/memos-local-plugin/core/embedding/index.ts @@ -19,6 +19,7 @@ export { l2Normalize, enforceDim, postProcess, toFloat32 } from "./normalize.js" export { createEmbeddingRetryWorker, systemErrorEvent } from "./retry-worker.js"; export type { EmbeddingRetryWorker } from "./retry-worker.js"; export type { + EmbedCallOptions, EmbedInput, EmbedRole, EmbedStats, diff --git a/apps/memos-local-plugin/core/embedding/providers/cohere.ts b/apps/memos-local-plugin/core/embedding/providers/cohere.ts index 891cd4506..58214d341 100644 --- a/apps/memos-local-plugin/core/embedding/providers/cohere.ts +++ b/apps/memos-local-plugin/core/embedding/providers/cohere.ts @@ -21,7 +21,7 @@ export class CohereEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "cohere"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -49,6 +49,8 @@ export class CohereEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/gemini.ts b/apps/memos-local-plugin/core/embedding/providers/gemini.ts index 91d97acba..68ba22708 100644 --- a/apps/memos-local-plugin/core/embedding/providers/gemini.ts +++ b/apps/memos-local-plugin/core/embedding/providers/gemini.ts @@ -22,7 +22,7 @@ export class GeminiEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "gemini"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -50,6 +50,8 @@ export class GeminiEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/mistral.ts b/apps/memos-local-plugin/core/embedding/providers/mistral.ts index 21f50769f..7eace8ab7 100644 --- a/apps/memos-local-plugin/core/embedding/providers/mistral.ts +++ b/apps/memos-local-plugin/core/embedding/providers/mistral.ts @@ -23,7 +23,7 @@ export class MistralEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "mistral"; async embed(texts: string[], _role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -46,6 +46,8 @@ export class MistralEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/openai.ts b/apps/memos-local-plugin/core/embedding/providers/openai.ts index c0df47831..57d852639 100644 --- a/apps/memos-local-plugin/core/embedding/providers/openai.ts +++ b/apps/memos-local-plugin/core/embedding/providers/openai.ts @@ -27,7 +27,7 @@ export class OpenAiEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "openai_compatible"; async embed(texts: string[], _role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -53,6 +53,8 @@ export class OpenAiEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/providers/voyage.ts b/apps/memos-local-plugin/core/embedding/providers/voyage.ts index f89eca832..6f36fe6d8 100644 --- a/apps/memos-local-plugin/core/embedding/providers/voyage.ts +++ b/apps/memos-local-plugin/core/embedding/providers/voyage.ts @@ -23,7 +23,7 @@ export class VoyageEmbeddingProvider implements EmbeddingProvider { readonly name: EmbeddingProviderName = "voyage"; async embed(texts: string[], role: EmbedRole, ctx: ProviderCallCtx): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.EMBEDDING_UNAVAILABLE, @@ -50,6 +50,8 @@ export class VoyageEmbeddingProvider implements EmbeddingProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/embedding/retry-worker.ts b/apps/memos-local-plugin/core/embedding/retry-worker.ts index 3620fd9e3..34db37bff 100644 --- a/apps/memos-local-plugin/core/embedding/retry-worker.ts +++ b/apps/memos-local-plugin/core/embedding/retry-worker.ts @@ -35,6 +35,7 @@ export function createEmbeddingRetryWorker( const workerId = `embedding-retry-${ids.span()}`; let timer: ReturnType | null = null; let running: Promise | null = null; + let stopped = false; async function runOnce(): Promise { if (!deps.embedder) return; @@ -103,6 +104,11 @@ export function createEmbeddingRetryWorker( const message = err instanceof Error ? err.message : String(err); const at = now(); const terminal = attemptNo >= job.maxAttempts; + const providerRetryAt = retryAtFromError(err, at); + const nextAttemptAt = Math.max( + at + backoffMs(attemptNo), + providerRetryAt ?? 0, + ); const recorded = terminal ? deps.repos.embeddingRetryQueue.markFailedClaimed(job.id, { ...claim, @@ -113,7 +119,7 @@ export function createEmbeddingRetryWorker( : deps.repos.embeddingRetryQueue.markRetryClaimed(job.id, { ...claim, attempts: attemptNo, - nextAttemptAt: at + backoffMs(attemptNo), + nextAttemptAt, error: message, now: at, }); @@ -121,7 +127,10 @@ export function createEmbeddingRetryWorker( deps.log.debug("embedding_retry.stale_failure_ignored", { jobId: job.id, terminal }); return; } - emitFailure(job, attemptNo, message, terminal, at); + emitFailure(job, attemptNo, message, terminal, at, { + providerRetryAt, + nextAttemptAt: terminal ? null : nextAttemptAt, + }); } } @@ -153,6 +162,7 @@ export function createEmbeddingRetryWorker( message: string, terminal: boolean, at: number, + retry: { providerRetryAt: number | null; nextAttemptAt: number | null }, ): void { const payload = { kind: "embedding.retry_failed", @@ -164,6 +174,8 @@ export function createEmbeddingRetryWorker( maxAttempts: job.maxAttempts, terminal, message, + providerRetryAt: retry.providerRetryAt, + nextAttemptAt: retry.nextAttemptAt, }; deps.log.warn("embedding_retry.failed", payload); try { @@ -182,7 +194,7 @@ export function createEmbeddingRetryWorker( } function tick(): void { - if (running) return; + if (stopped || running) return; running = runOnce().finally(() => { running = null; }); @@ -190,21 +202,30 @@ export function createEmbeddingRetryWorker( return { start(): void { - if (timer || !deps.embedder) return; + if (stopped || timer || !deps.embedder) return; tick(); timer = setInterval(tick, deps.intervalMs ?? DEFAULT_INTERVAL_MS); }, stop(): void { + stopped = true; if (timer) clearInterval(timer); timer = null; }, async flush(): Promise { - tick(); + if (!stopped) tick(); if (running) await running; }, }; } +function retryAtFromError(err: unknown, nowMs: number): number | null { + if (!err || typeof err !== "object") return null; + const details = (err as { details?: unknown }).details; + if (!details || typeof details !== "object") return null; + const retryAt = Number((details as { retryAt?: unknown }).retryAt); + return Number.isSafeInteger(retryAt) && retryAt > nowMs ? retryAt : null; +} + function backoffMs(attemptNo: number): number { return Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** Math.max(0, attemptNo - 1)); } diff --git a/apps/memos-local-plugin/core/embedding/types.ts b/apps/memos-local-plugin/core/embedding/types.ts index 4f3f5eb99..95726703c 100644 --- a/apps/memos-local-plugin/core/embedding/types.ts +++ b/apps/memos-local-plugin/core/embedding/types.ts @@ -6,6 +6,7 @@ */ import type { EmbeddingVector } from "../types.js"; +import type { RetryDiagnosticDetails } from "../util/retry-after.js"; // ─── Config ────────────────────────────────────────────────────────────────── @@ -62,7 +63,7 @@ export interface EmbeddingConfig { onStatus?: (detail: EmbeddingStatusDetail) => void; } -export interface EmbeddingErrorDetail { +export interface EmbeddingErrorDetail extends RetryDiagnosticDetails { kind: "embedding"; provider: EmbeddingProviderName | string; model: string; @@ -73,7 +74,7 @@ export interface EmbeddingErrorDetail { at?: number; } -export interface EmbeddingStatusDetail { +export interface EmbeddingStatusDetail extends RetryDiagnosticDetails { kind: "embedding"; status: "ok" | "error"; provider: EmbeddingProviderName | string; @@ -128,6 +129,8 @@ export interface ProviderCallCtx { log: ProviderLogger; /** AbortSignal honored across HTTP + native calls. */ signal?: AbortSignal; + /** Absolute end-to-end deadline shared across provider retry attempts. */ + deadlineAt?: number; } export interface ProviderLogger { @@ -166,13 +169,16 @@ export interface Embedder { /** Model identifier as configured by the operator (e.g. "bge-m3"). */ readonly model: string; - embedOne(input: string | EmbedInput): Promise; + embedOne(input: string | EmbedInput, options?: EmbedCallOptions): Promise; /** * Batch-embed many texts. Results keep input order. Duplicates are deduped * internally so a text repeated N times causes 1 cache miss max. */ - embedMany(inputs: Array): Promise; + embedMany( + inputs: Array, + options?: EmbedCallOptions, + ): Promise; stats(): EmbedStats; @@ -181,6 +187,12 @@ export interface Embedder { close(): Promise; } +export interface EmbedCallOptions { + signal?: AbortSignal; + /** Absolute end-to-end deadline shared across provider retry attempts. */ + deadlineAt?: number; +} + // ─── Errors ────────────────────────────────────────────────────────────────── export interface ProviderHttpFailure { diff --git a/apps/memos-local-plugin/core/index.ts b/apps/memos-local-plugin/core/index.ts index d2b979bce..c5e4fab9d 100644 --- a/apps/memos-local-plugin/core/index.ts +++ b/apps/memos-local-plugin/core/index.ts @@ -110,6 +110,7 @@ export { MistralEmbeddingProvider, type EmbedCache, type EmbedCacheStats, + type EmbedCallOptions, type EmbedInput, type EmbedRole, type EmbedStats, diff --git a/apps/memos-local-plugin/core/llm/client.ts b/apps/memos-local-plugin/core/llm/client.ts index 6bedafa70..ee456ac12 100644 --- a/apps/memos-local-plugin/core/llm/client.ts +++ b/apps/memos-local-plugin/core/llm/client.ts @@ -18,6 +18,7 @@ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; import { rootLogger } from "../logger/index.js"; import type { Logger } from "../logger/types.js"; +import { extractRetryDiagnostics } from "../util/retry-after.js"; import { getHostLlmBridge } from "./host-bridge.js"; import { buildJsonSystemHint, parseLlmJson } from "./json-mode.js"; import { AnthropicLlmProvider } from "./providers/anthropic.js"; @@ -292,6 +293,7 @@ export function createLlmClientWithProvider( }, log: pLog, signal: opts?.signal, + deadlineAt: opts?.deadlineAt, }; } @@ -370,6 +372,7 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(hostErr), code: hostErr instanceof MemosError ? hostErr.code : undefined, + ...extractRetryDiagnostics(hostErr instanceof MemosError ? hostErr.details : undefined), at: failAt, durationMs: Date.now() - startedAt, fallbackProvider: "host", @@ -395,6 +398,7 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), at: failAt, durationMs: Date.now() - startedAt, op, @@ -425,6 +429,7 @@ export function createLlmClientWithProvider( message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, at: Date.now(), + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), }); } catch { /* sink errors are non-fatal */ @@ -444,6 +449,10 @@ export function createLlmClientWithProvider( op?: string; episodeId?: string; phase?: string; + retryAfterMs?: number; + retryAt?: number; + retryDecision?: "wait" | "defer" | "stop"; + retryReason?: string; }): void { if (!config.onStatus) return; try { @@ -616,6 +625,7 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(err), code: err instanceof MemosError ? err.code : undefined, + ...extractRetryDiagnostics(err instanceof MemosError ? err.details : undefined), at: failAt, durationMs: Date.now() - start, op: opts?.op ?? "stream", @@ -669,6 +679,7 @@ export function createLlmClientWithProvider( model: config.model, message: summarizeErrMessage(primaryErr), code: primaryErr instanceof MemosError ? primaryErr.code : undefined, + ...extractRetryDiagnostics(primaryErr instanceof MemosError ? primaryErr.details : undefined), at: fallbackAt, durationMs: completion.durationMs, fallbackProvider: "host", diff --git a/apps/memos-local-plugin/core/llm/fetcher.ts b/apps/memos-local-plugin/core/llm/fetcher.ts index 53eb55ec3..358c73275 100644 --- a/apps/memos-local-plugin/core/llm/fetcher.ts +++ b/apps/memos-local-plugin/core/llm/fetcher.ts @@ -11,6 +11,15 @@ */ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; +import { + getRetryCooldown, + parseRetryAfterMs, + planRetry, + recordRetryCooldown, + retryCooldownKey, + type RetryPlan, + waitForRetry, +} from "../util/retry-after.js"; import type { LlmProviderLogger, LlmProviderName } from "./types.js"; export interface HttpPostOpts { @@ -20,6 +29,10 @@ export interface HttpPostOpts { timeoutMs: number; maxRetries: number; signal?: AbortSignal; + /** Absolute end-to-end deadline. Unlike timeoutMs, this is not renewed per attempt. */ + deadlineAt?: number; + /** Model/deployment scope; prevents one model cooldown from blocking another. */ + cooldownScope?: string; provider: LlmProviderName; log: LlmProviderLogger; onRetry?: (attempt: number) => void; @@ -35,11 +48,33 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< }> { let attempt = 0; let lastErr: unknown = null; + const cooldownKey = retryCooldownKey("llm", opts.provider, opts.url, opts.cooldownScope); while (attempt <= opts.maxRetries) { attempt++; const start = Date.now(); try { + const cooldown = getRetryCooldown(cooldownKey, start); + if (cooldown) { + const details = { + provider: opts.provider, + url: opts.url, + status: cooldown.status, + attempt, + maxRetries: opts.maxRetries, + retryAfterMs: cooldown.retryAfterMs, + retryAt: cooldown.retryAt, + retryDecision: "defer", + retryReason: "cooldown_active", + remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt, start), + }; + opts.log.warn("http.retry_cooldown", details); + throw new MemosError( + errCodeForStatus(cooldown.status), + `${opts.provider} is cooling down until ${new Date(cooldown.retryAt).toISOString()}`, + details, + ); + } const signal = mergeSignals(opts.signal, AbortSignal.timeout(opts.timeoutMs)); const resp = await fetch(opts.url, { method: "POST", @@ -56,22 +91,63 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< if (!resp.ok) { const text = await safeText(resp); const transient = resp.status >= 500 || resp.status === 429; + const retryAfterMs = resp.status === 429 || resp.status === 503 + ? parseRetryAfterMs(resp.headers.get("Retry-After")) + : null; + if (retryAfterMs !== null) { + recordRetryCooldown(cooldownKey, { + retryAfterMs, + retryAt: Date.now() + retryAfterMs, + status: resp.status, + }); + } opts.log.warn("http.non_ok", { status: resp.status, attempt, transient, durationMs: ms, + retryAfterMs, body: truncateLogBody(text), }); if (transient && attempt <= opts.maxRetries) { + const plan = planRetry({ + attempt, + baseMs: 250, + jitterMaxMs: 120, + retryAfterMs, + deadlineAt: opts.deadlineAt, + }); + const retryDetails = retryPlanDetails(plan, opts, resp.status, attempt); + if (plan.action === "defer") { + opts.log.warn("http.retry_deferred", retryDetails); + throw new MemosError( + errCodeForStatus(resp.status), + `HTTP ${resp.status} from ${opts.provider}; retry deferred until ${new Date(plan.retryAt).toISOString()}`, + retryDetails, + ); + } + opts.log.warn("http.retry_scheduled", retryDetails); opts.onRetry?.(attempt); - await backoff(attempt); + await waitForRetry(plan.delayMs, opts.signal); continue; } throw new MemosError( errCodeForStatus(resp.status), `HTTP ${resp.status} from ${opts.provider}`, - { provider: opts.provider, url: opts.url, status: resp.status, body: text }, + { + provider: opts.provider, + url: opts.url, + status: resp.status, + body: text, + ...(retryAfterMs === null + ? {} + : { + retryAfterMs, + retryAt: Date.now() + retryAfterMs, + retryDecision: "stop", + retryReason: "retries_exhausted", + }), + }, ); } @@ -85,8 +161,15 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< } catch (err) { lastErr = err; if (err instanceof MemosError) throw err; + if (opts.signal?.aborted) { + throw new MemosError( + ERROR_CODES.LLM_TIMEOUT, + `${opts.provider} request was cancelled`, + { provider: opts.provider, url: opts.url, cancelled: true }, + ); + } const transient = isTransientError(err); - const timedOut = isTimeout(err); + const timedOut = isTimeout(err) || opts.signal?.aborted === true; opts.log.warn("http.exception", { attempt, transient, @@ -94,8 +177,24 @@ export async function httpPostJson(opts: HttpPostOpts): Promise< err: toErrDetail(err), }); if ((transient || timedOut) && attempt <= opts.maxRetries) { + const plan = planRetry({ + attempt, + baseMs: 250, + jitterMaxMs: 120, + deadlineAt: opts.deadlineAt, + }); + const retryDetails = retryPlanDetails(plan, opts, null, attempt); + if (plan.action === "defer") { + opts.log.warn("http.retry_deferred", retryDetails); + throw new MemosError( + timedOut ? ERROR_CODES.LLM_TIMEOUT : ERROR_CODES.LLM_UNAVAILABLE, + `${opts.provider} retry cannot fit the request deadline`, + retryDetails, + ); + } + opts.log.warn("http.retry_scheduled", retryDetails); opts.onRetry?.(attempt); - await backoff(attempt); + await waitForRetry(plan.delayMs, opts.signal); continue; } if (timedOut) { @@ -245,11 +344,31 @@ function isTimeout(err: unknown): boolean { return false; } -async function backoff(attempt: number): Promise { - const base = 250; - const jitter = Math.floor(Math.random() * 120); - const ms = base * 2 ** (attempt - 1) + jitter; - await new Promise((r) => setTimeout(r, ms)); +function retryPlanDetails( + plan: RetryPlan, + opts: HttpPostOpts, + status: number | null, + attempt: number, +): Record { + return { + provider: opts.provider, + url: opts.url, + status, + attempt, + maxRetries: opts.maxRetries, + backoffMs: plan.backoffMs, + plannedDelayMs: plan.delayMs, + retryAfterMs: plan.retryAfterMs, + retryAt: plan.retryAt, + retrySource: plan.source, + retryDecision: plan.action, + ...(plan.action === "defer" ? { retryReason: plan.reason } : {}), + remainingDeadlineMs: remainingDeadlineMs(opts.deadlineAt), + }; +} + +function remainingDeadlineMs(deadlineAt?: number, nowMs: number = Date.now()): number | null { + return deadlineAt === undefined ? null : Math.max(0, deadlineAt - nowMs); } function mergeSignals(a: AbortSignal | undefined, b: AbortSignal): AbortSignal { diff --git a/apps/memos-local-plugin/core/llm/providers/anthropic.ts b/apps/memos-local-plugin/core/llm/providers/anthropic.ts index 66a9ee446..6c510ef75 100644 --- a/apps/memos-local-plugin/core/llm/providers/anthropic.ts +++ b/apps/memos-local-plugin/core/llm/providers/anthropic.ts @@ -31,7 +31,7 @@ export class AnthropicLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -68,6 +68,8 @@ export class AnthropicLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/bedrock.ts b/apps/memos-local-plugin/core/llm/providers/bedrock.ts index 7c00470e7..d956b7111 100644 --- a/apps/memos-local-plugin/core/llm/providers/bedrock.ts +++ b/apps/memos-local-plugin/core/llm/providers/bedrock.ts @@ -36,7 +36,7 @@ export class BedrockLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.endpoint || config.endpoint.length === 0) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -85,6 +85,8 @@ export class BedrockLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/gemini.ts b/apps/memos-local-plugin/core/llm/providers/gemini.ts index 4e6273b55..6bfa323eb 100644 --- a/apps/memos-local-plugin/core/llm/providers/gemini.ts +++ b/apps/memos-local-plugin/core/llm/providers/gemini.ts @@ -39,7 +39,7 @@ export class GeminiLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; if (!config.apiKey) { throw new MemosError( ERROR_CODES.LLM_UNAVAILABLE, @@ -59,6 +59,8 @@ export class GeminiLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/providers/openai.ts b/apps/memos-local-plugin/core/llm/providers/openai.ts index d8a5a20af..562521756 100644 --- a/apps/memos-local-plugin/core/llm/providers/openai.ts +++ b/apps/memos-local-plugin/core/llm/providers/openai.ts @@ -51,7 +51,7 @@ export class OpenAiLlmProvider implements LlmProvider { opts: ProviderCallInput, ctx: LlmProviderCtx, ): Promise { - const { config, log, signal } = ctx; + const { config, log, signal, deadlineAt } = ctx; const url = normalizeEndpoint( config.endpoint && config.endpoint.length > 0 ? config.endpoint @@ -93,6 +93,8 @@ export class OpenAiLlmProvider implements LlmProvider { timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, + deadlineAt, + cooldownScope: config.model, provider: this.name, log, }); diff --git a/apps/memos-local-plugin/core/llm/types.ts b/apps/memos-local-plugin/core/llm/types.ts index a37a2677d..2dd761f32 100644 --- a/apps/memos-local-plugin/core/llm/types.ts +++ b/apps/memos-local-plugin/core/llm/types.ts @@ -6,6 +6,7 @@ */ import type { ReasoningConfig as ConfigReasoningConfig } from "../config/schema.js"; +import type { RetryDiagnosticDetails } from "../util/retry-after.js"; // ─── Providers & config ────────────────────────────────────────────────────── @@ -88,7 +89,7 @@ export interface LlmCircuitBreakerConfig { now?: () => number; } -export interface LlmErrorDetail { +export interface LlmErrorDetail extends RetryDiagnosticDetails { provider: LlmProviderName | string; model: string; message: string; @@ -105,7 +106,7 @@ export interface LlmErrorDetail { role?: "llm" | "skillEvolver"; } -export interface LlmStatusDetail { +export interface LlmStatusDetail extends RetryDiagnosticDetails { status: "ok" | "fallback" | "error" | "circuit_open"; provider: LlmProviderName | string; model: string; @@ -149,6 +150,8 @@ export interface LlmCallOptions { maxTokens?: number; /** Per-call timeout. */ timeoutMs?: number; + /** Absolute end-to-end deadline shared across provider retry attempts. */ + deadlineAt?: number; /** AbortSignal honored across HTTP + host-bridge calls. */ signal?: AbortSignal; /** @@ -213,6 +216,8 @@ export interface LlmProviderCtx { log: LlmProviderLogger; /** Call abort signal; providers must honor it. */ signal?: AbortSignal; + /** Absolute end-to-end deadline; providers must not renew it per retry. */ + deadlineAt?: number; } export interface LlmProviderLogger { diff --git a/apps/memos-local-plugin/core/pipeline/deps.ts b/apps/memos-local-plugin/core/pipeline/deps.ts index 8a0772119..79714b35e 100644 --- a/apps/memos-local-plugin/core/pipeline/deps.ts +++ b/apps/memos-local-plugin/core/pipeline/deps.ts @@ -101,6 +101,10 @@ import type { import { wrapRetrievalRepos } from "./retrieval-repos.js"; import { createSemaphore } from "../util/semaphore.js"; import { rateLimitLlmClient } from "../util/rate-limited-llm.js"; +import { + prioritizeEmbedder, + type ForegroundResources, +} from "../util/foreground-resources.js"; // ─── Algorithm config slice helper ──────────────────────────────────────── @@ -208,19 +212,23 @@ export function buildPipelineSubscribers( buses: PipelineBuses, algorithm: PipelineAlgorithmConfig, session?: PipelineSessionSet, + resources?: ForegroundResources, ): PipelineSubscriberSet { const log = deps.log ?? rootLogger.child({ channel: "core.pipeline" }); const bgLlmSemaphore = createSemaphore(algorithm.session.bgLlmConcurrency); - const bgLlm = rateLimitLlmClient(deps.llm, bgLlmSemaphore); - const bgReflectLlm = rateLimitLlmClient(deps.reflectLlm, bgLlmSemaphore); - const bgL3Llm = rateLimitLlmClient(deps.l3Llm ?? deps.llm, bgLlmSemaphore); + const bgLlm = rateLimitLlmClient(deps.llm, bgLlmSemaphore, resources); + const bgReflectLlm = rateLimitLlmClient(deps.reflectLlm, bgLlmSemaphore, resources); + const bgL3Llm = rateLimitLlmClient(deps.l3Llm ?? deps.llm, bgLlmSemaphore, resources); + const bgEmbedder = resources + ? prioritizeEmbedder(deps.embedder, resources, "background") + : deps.embedder; const lightweightMode = algorithm.lightweightMemory.enabled; const captureRunner = createCaptureRunner({ tracesRepo: deps.repos.traces, embeddingRetryQueue: deps.repos.embeddingRetryQueue, episodesRepo: adaptEpisodesRepo(deps.repos.episodes), - embedder: deps.embedder, + embedder: bgEmbedder, llm: bgLlm, // Issue #2148: capture batch reflection emits JSON, so it must use // the main model rather than the potentially thinking-enabled @@ -327,7 +335,7 @@ export function buildPipelineSubscribers( const skillHandle = attachSkillSubscriber({ repos: deps.repos, - embedder: deps.embedder, + embedder: bgEmbedder, llm: bgLlm, bus: buses.skill, l2Bus: buses.l2, @@ -339,7 +347,7 @@ export function buildPipelineSubscribers( const feedbackHandle = attachFeedbackSubscriber({ repos: deps.repos, llm: bgLlm, - embedder: deps.embedder, + embedder: bgEmbedder, bus: buses.feedback, log: log.child({ channel: "core.feedback" }), config: algorithm.feedback, @@ -404,14 +412,17 @@ export function buildPipelineSession( export function buildRetrievalDeps( deps: PipelineDeps, algorithm: PipelineAlgorithmConfig, + resources?: ForegroundResources, ): RetrievalDeps { - const embedder = deps.embedder; + const embedder = resources + ? prioritizeEmbedder(deps.embedder, resources, "foreground") + : deps.embedder; return { repos: wrapRetrievalRepos(deps.repos, deps.namespace), embedder: embedder ? { - embed: (text, role) => - embedder.embedOne({ text, role: role ?? "query" }), + embed: (text, role, options) => + embedder.embedOne({ text, role: role ?? "query" }, options), } : { // Degraded mode: empty vector so vector-scoring falls back to diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index c9254e092..a0354bf64 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -1232,18 +1232,59 @@ export function createMemoryCore( const statsLine = `phase=${phase}, stored=${storedCount}` + (r.warnings.length > 0 ? `, warnings=${r.warnings.length}` : ""); - const details = r.traces.map((tc) => ({ - role: inferTurnRole(tc), - action: phase === "lite" ? ("stored" as const) : ("reflected" as const), - summary: tc.reflection?.text ?? null, - content: ( - tc.userText || - tc.agentText || - summarizeToolCalls(tc.toolCalls) || - "" - ).slice(0, 400), - traceId: tc.traceId, - })); + const action = phase === "lite" + ? ("stored" as const) + : ("reflected" as const); + const details = r.traces.flatMap((tc) => { + const items: Array<{ + role: "user" | "assistant" | "tool" | "reflection" | "other"; + action: typeof action; + summary: string | null; + content: string; + traceId: string; + }> = []; + + if (tc.userText) { + items.push({ + role: "user", + action, + summary: null, + content: tc.userText.slice(0, 400), + traceId: tc.traceId, + }); + } + if (tc.agentText) { + items.push({ + role: "assistant", + action, + summary: null, + content: tc.agentText.slice(0, 400), + traceId: tc.traceId, + }); + } + + const toolSummary = summarizeToolCalls(tc.toolCalls); + if (items.length === 0) { + items.push({ + role: toolSummary ? "tool" : "other", + action, + summary: tc.reflection?.text ?? null, + content: toolSummary.slice(0, 400), + traceId: tc.traceId, + }); + } else if (tc.reflection?.text) { + // Keep the existing reflect-phase summary visible without + // presenting it as either side's original chat content. + items.push({ + role: "reflection", + action, + summary: tc.reflection.text, + content: "", + traceId: tc.traceId, + }); + } + return items; + }); handle.repos.apiLogs.insert({ toolName: "memory_add", input: { @@ -6195,26 +6236,3 @@ function summarizeToolCalls( }) .join("\n"); } - -/** - * Heuristic role inference for api_logs "memory_add" rows — mirrors - * the legacy plugin's behaviour where each captured turn showed up - * labelled `user` / `assistant` / `tool` on the Logs page. - * - * Priority: if the step carries userText (the user's query), label it - * "user" even when toolCalls are present — this is the first sub-step - * of a multi-tool turn and semantically represents the user request. - */ -function inferTurnRole(step: { - userText?: string; - agentText?: string; - toolCalls?: readonly unknown[]; -}): "user" | "assistant" | "tool" | "other" { - const u = (step.userText ?? "").length; - const a = (step.agentText ?? "").length; - if (u > 0 && (step.toolCalls?.length ?? 0) > 0) return "user"; - if ((step.toolCalls?.length ?? 0) > 0) return "tool"; - if (u >= a && u > 0) return "user"; - if (a > 0) return "assistant"; - return "other"; -} diff --git a/apps/memos-local-plugin/core/pipeline/orchestrator.ts b/apps/memos-local-plugin/core/pipeline/orchestrator.ts index ac48133ad..4ed0427d1 100644 --- a/apps/memos-local-plugin/core/pipeline/orchestrator.ts +++ b/apps/memos-local-plugin/core/pipeline/orchestrator.ts @@ -83,29 +83,39 @@ import { onBroadcastLog } from "../logger/transports/sse-broadcast.js"; import { createEmbeddingRetryWorker, systemErrorEvent } from "../embedding/index.js"; import type { EpisodeSnapshot } from "../session/index.js"; import type { IntentDecision, RelationDecision, TurnRelation } from "../session/types.js"; +import { + createForegroundResources, + prioritizeEmbedder, +} from "../util/foreground-resources.js"; +import { createRequestDeadline } from "../util/request-deadline.js"; function classifyWithTimeout( classifyFn: () => Promise, timeoutMs: number, log: Logger, ): Promise { + let timer: ReturnType | null = null; return Promise.race([ classifyFn(), - new Promise((_, reject) => - setTimeout(() => reject(new Error("classify_timeout")), timeoutMs), - ), - ]).catch((err) => { - log.warn("relation.classify_timeout", { - timeoutMs, - err: err instanceof Error ? err.message : String(err), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("classify_timeout")), timeoutMs); + }), + ]) + .catch((err) => { + log.warn("relation.classify_timeout", { + timeoutMs, + err: err instanceof Error ? err.message : String(err), + }); + return { + relation: "follow_up" as const, + confidence: 0, + reason: "classify_timeout", + signals: ["classify_timeout"], + }; + }) + .finally(() => { + if (timer) clearTimeout(timer); }); - return { - relation: "new_task" as const, - confidence: 0, - reason: "classify_timeout", - signals: ["classify_timeout"], - }; - }); } // ─── Factory ────────────────────────────────────────────────────────────── @@ -115,6 +125,12 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { const algorithm = extractAlgorithmConfig(deps); const lightweightMode = algorithm.lightweightMemory.enabled; const buses = buildPipelineBuses(); + const foregroundResources = createForegroundResources(); + const backgroundEmbedder = prioritizeEmbedder( + deps.embedder, + foregroundResources, + "background", + ); // Session + intent. const session = buildPipelineSession(deps, buses.session); @@ -123,7 +139,13 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { // Pass `session` so the reward runner's `getEpisodeSnapshot` hook // can resolve the live, in-memory episode (with turns populated) // rather than falling back to the empty row from SQLite. - const subs = buildPipelineSubscribers(deps, buses, algorithm, session); + const subs = buildPipelineSubscribers( + deps, + buses, + algorithm, + session, + foregroundResources, + ); // Core-event aggregator. Every internal bus funnels into one stream. const eventListeners = new Set<(e: CoreEvent) => void>(); @@ -160,7 +182,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { let retryEventSeq = 1_000_000; const embeddingRetryWorker = createEmbeddingRetryWorker({ repos: deps.repos, - embedder: deps.embedder, + embedder: backgroundEmbedder, log: log.child({ channel: "core.embedding.retry" }), now: deps.now, onSystemError: (payload, correlationId) => { @@ -366,6 +388,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userText: string, meta: Record, turnTs?: number, + signal?: AbortSignal, ): Promise { const currentEpId = openEpisodeBySession.get(sessionId); if (currentEpId) { @@ -387,6 +410,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: lightweightEpisodeMeta(meta), + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return snap; @@ -433,13 +457,20 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userText: string, meta: Record, agent: AgentKind, + signal?: AbortSignal, ): Promise<{ episode: EpisodeSnapshot; sessionId: SessionId; relation?: string }> { const mergeMode = algorithm.session.followUpMode === "merge_follow_ups"; const mergeCapMs = algorithm.session.mergeMaxGapMs; const turnTs = timestampFromMeta(meta, "startedAtTurnTs"); if (lightweightMode) { - const snap = await startLightweightEpisode(sessionId, userText, meta, turnTs); + const snap = await startLightweightEpisode( + sessionId, + userText, + meta, + turnTs, + signal, + ); return { episode: snap, sessionId, relation: "lightweight_memory" }; } @@ -465,6 +496,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: currentEpId, + signal, }), algorithm.session.classifyTimeoutMs, log, @@ -582,6 +614,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: "new_task" }, + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -601,6 +634,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: decision.relation, gapMs }, + signal, }); openEpisodeBySession.set(sessionId, fresh.id as EpisodeId); return { episode: fresh, sessionId, relation: decision.relation }; @@ -645,6 +679,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: snapshot.id as EpisodeId, + signal, }), algorithm.session.classifyTimeoutMs, log, @@ -748,6 +783,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta, + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: "bootstrap" }; @@ -762,6 +798,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { newUserText: userText, gapMs, prevEpisodeId: prev.episodeId, + signal, }), algorithm.session.classifyTimeoutMs, log, @@ -861,6 +898,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: "new_task" }, + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -871,6 +909,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { userMessage: userText, ts: turnTs, meta: { ...meta, relation: decision.relation }, + signal, }); openEpisodeBySession.set(sessionId, snap.id as EpisodeId); return { episode: snap, sessionId, relation: decision.relation }; @@ -1043,7 +1082,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { // ─── Retrieval entry points ───────────────────────────────────────────── - const retrievalDeps = buildRetrievalDeps(deps, algorithm); + const retrievalDeps = buildRetrievalDeps(deps, algorithm, foregroundResources); const turnStartRetrievalStats = new Map(); function retrievalDepsFor(namespace = deps.namespace): typeof retrievalDeps { @@ -1057,6 +1096,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { async function retrieveTurnStart( input: TurnInputDTO, plan?: RetrievePlan, + signal?: AbortSignal, ): Promise { const ctx = { reason: "turn_start" as const, @@ -1073,6 +1113,8 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { { events: buses.retrieval, skipLlmFilter: input.contextHints?.__memosDeferLlmFilterToCaller === true, + signal, + deadlineAt: input.deadlineAt, plan: plan ? { scenarioId: plan.scenarioId, @@ -1175,13 +1217,45 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { } async function onTurnStartOnce(input: TurnInputDTO): Promise { + const leaveForeground = foregroundResources.enterForeground(); + const deadline = + input.deadlineAt === undefined + ? null + : createRequestDeadline(input.deadlineAt); + const startedAt = Date.now(); + let stage = "ensure_session"; + try { + return await onTurnStartForeground(input, deadline?.signal, (next) => { + stage = next; + }); + } finally { + if (deadline?.signal.aborted) { + log.warn("turn.start.deadline_exceeded", { + sessionId: input.sessionId, + deadlineAt: input.deadlineAt, + elapsedMs: Date.now() - startedAt, + stage, + }); + } + deadline?.dispose(); + leaveForeground(); + } + } + + async function onTurnStartForeground( + input: TurnInputDTO, + signal?: AbortSignal, + setStage: (stage: string) => void = () => {}, + ): Promise { const t0 = now(); + setStage("ensure_session"); const initialSessionId = await ensureSession( input.agent, input.sessionId, input.contextHints, ); + setStage("relation_and_episode"); const routing = await openEpisodeIfNeeded( initialSessionId, input.userText, @@ -1192,6 +1266,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { startedAtTurnTs: input.ts, }, input.agent, + signal, ); const sessionId = routing.sessionId; @@ -1203,10 +1278,12 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { sessionId, episodeId: episode.id as EpisodeId, }; + setStage("intent"); const schedulerIntent = await intentForCurrentTurn({ episode, userText: input.userText, ts: input.ts, + signal, }); const retrievePlan = scheduleInjection({ userText: input.userText, @@ -1240,10 +1317,13 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { retrievalTotalMs: 0, elapsedMs: now() - t0, }); + setStage("complete"); return packet; } - const packet = await retrieveTurnStart(normalized, retrievePlan); + setStage("retrieval"); + const packet = await retrieveTurnStart(normalized, retrievePlan, signal); + setStage("complete"); // Always stamp the routed sessionId + episodeId on the packet so // adapters can correlate the subsequent `agent_end` / `turn.end` // call without needing a separate round-trip to the session @@ -1502,12 +1582,28 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { async function shutdown(reason: string = "shutdown"): Promise { log.info("pipeline.shutdown.begin", { reason }); + // Stop admitting retry jobs, but preserve a bounded grace period for raw + // capture and downstream enrichment. Hermes' bridge owns a 20s outer + // shutdown ceiling, so abort before that rather than either hanging or + // discarding every single-shot session's enrichment immediately. + embeddingRetryWorker.stop(); + const flushPromise = flush(); try { - await flush(); + const completed = await settlesWithin(flushPromise, 15_000); + if (!completed) { + log.warn("pipeline.flush_timeout", { reason, timeoutMs: 15_000 }); + foregroundResources.shutdown(reason); + const aborted = await settlesWithin(flushPromise, 4_000); + if (!aborted) { + log.warn("pipeline.flush_abandoned", { reason, abortWaitMs: 4_000 }); + } + } } catch (err) { log.warn("pipeline.flush_failed", { err: err instanceof Error ? err.message : String(err), }); + } finally { + foregroundResources.shutdown(reason); } // Detach subscribers — prevents late events from re-queuing work. subs.subscriptions.capture.stop(); @@ -1516,13 +1612,26 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { subs.l3.detach(); subs.skills.dispose(); subs.feedback.dispose(); - embeddingRetryWorker.stop(); bridge.dispose(); logSubscription(); session.sessionManager.shutdown(reason); log.info("pipeline.shutdown.done", { reason }); } + async function settlesWithin(promise: Promise, timeoutMs: number): Promise { + let timer: ReturnType | null = null; + try { + return await Promise.race([ + promise.then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), timeoutMs); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } + } + function now(): number { return (deps.now ?? Date.now)(); } @@ -1589,6 +1698,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { episode: EpisodeSnapshot; userText: string; ts?: number; + signal?: AbortSignal; }): Promise { const firstTurn = input.episode.turns[0]; const isFreshEpisodeForThisTurn = @@ -1603,6 +1713,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { return session.intent.classify(input.userText, { episodeId: input.episode.id as EpisodeId, + signal: input.signal, }); } diff --git a/apps/memos-local-plugin/core/retrieval/llm-filter.ts b/apps/memos-local-plugin/core/retrieval/llm-filter.ts index 142bd626d..f665cb954 100644 --- a/apps/memos-local-plugin/core/retrieval/llm-filter.ts +++ b/apps/memos-local-plugin/core/retrieval/llm-filter.ts @@ -50,6 +50,8 @@ export interface FilterDeps { llm: LlmClient | null; log: Logger; timeoutMs?: number; + deadlineAt?: number; + signal?: AbortSignal; config: Pick< RetrievalConfig, | "llmFilterEnabled" @@ -117,6 +119,10 @@ export async function llmFilterCandidates( if (!deps.llm) { return passthrough(ranked, "no_llm"); } + if (deps.signal?.aborted) { + deps.log.debug("llm_filter.deadline_exceeded", { candidateCount: ranked.length }); + return safeCutoff(ranked, deps); + } const bodyChars = deps.config.llmFilterCandidateBodyChars ?? DEFAULT_CANDIDATE_BODY_CHARS; @@ -148,6 +154,8 @@ ${list}`, episodeId: input.episodeId, temperature: 0, timeoutMs: deps.timeoutMs, + deadlineAt: deps.deadlineAt, + signal: deps.signal, // Output is only ordered indices + one bool, but the list can // legitimately be as long as the ranked candidates. maxTokens: filterOutputTokenBudget(ranked.length), diff --git a/apps/memos-local-plugin/core/retrieval/retrieve.ts b/apps/memos-local-plugin/core/retrieval/retrieve.ts index cec076f62..ff95debf4 100644 --- a/apps/memos-local-plugin/core/retrieval/retrieve.ts +++ b/apps/memos-local-plugin/core/retrieval/retrieve.ts @@ -75,6 +75,10 @@ export interface RetrieveOptions { * one unified final LLM filter across all routes. */ skipLlmFilter?: boolean; + /** Shared foreground cancellation signal. */ + signal?: AbortSignal; + /** Absolute request deadline used to cap optional LLM filtering. */ + deadlineAt?: number; } export interface RetrievePlanOverride { @@ -258,22 +262,28 @@ async function runAll( degraded: false, }; const queryVec = compiled.text - ? await deps.embedder.embed(compiled.text, "query").then((vec) => { - embeddingStats.ok = true; - return vec; - }).catch((err) => { - const code = (err as { code?: string })?.code; - const message = err instanceof Error ? err.message : String(err); - embeddingStats.degraded = true; - embeddingStats.errorCode = code; - embeddingStats.errorMessage = message; - log.warn("embed_failed", { - reason: ctx.reason, - code, - err: message, - }); - return null; - }) + ? await deps.embedder + .embed(compiled.text, "query", { + signal: opts.signal, + deadlineAt: opts.deadlineAt, + }) + .then((vec) => { + embeddingStats.ok = true; + return vec; + }) + .catch((err) => { + const code = (err as { code?: string })?.code; + const message = err instanceof Error ? err.message : String(err); + embeddingStats.degraded = true; + embeddingStats.errorCode = code; + embeddingStats.errorMessage = message; + log.warn("embed_failed", { + reason: ctx.reason, + code, + err: message, + }); + return null; + }) : null; // The keyword channels (FTS + pattern) work even without an embedder, @@ -415,6 +425,9 @@ async function runAll( llm: deps.llm ?? null, log, config: deps.config, + signal: opts.signal, + deadlineAt: opts.deadlineAt, + timeoutMs: filterTimeoutMs(opts.deadlineAt), }, ); @@ -472,6 +485,9 @@ async function runAll( llm: deps.llm ?? null, log, config: deps.config, + signal: opts.signal, + deadlineAt: opts.deadlineAt, + timeoutMs: filterTimeoutMs(opts.deadlineAt), }, ); @@ -668,6 +684,11 @@ async function runAll( } } +function filterTimeoutMs(deadlineAt?: number): number | undefined { + if (deadlineAt === undefined) return undefined; + return Math.max(1, Math.min(2_000, deadlineAt - Date.now())); +} + function emptyResult( reason: RetrievalReason, agent: AgentKind, diff --git a/apps/memos-local-plugin/core/retrieval/types.ts b/apps/memos-local-plugin/core/retrieval/types.ts index 8ed24935e..fe700c611 100644 --- a/apps/memos-local-plugin/core/retrieval/types.ts +++ b/apps/memos-local-plugin/core/retrieval/types.ts @@ -679,7 +679,11 @@ export interface RetrievalRepos { /** Abstract embedder surface consumed by retrieval. Mirrors `Embedder`. */ export interface RetrievalEmbedder { - embed: (text: string, role?: "query" | "document") => Promise; + embed: ( + text: string, + role?: "query" | "document", + options?: { signal?: AbortSignal; deadlineAt?: number }, + ) => Promise; } export interface RetrievalDeps { diff --git a/apps/memos-local-plugin/core/session/intent-classifier.ts b/apps/memos-local-plugin/core/session/intent-classifier.ts index 41206184d..a4896e07b 100644 --- a/apps/memos-local-plugin/core/session/intent-classifier.ts +++ b/apps/memos-local-plugin/core/session/intent-classifier.ts @@ -46,6 +46,8 @@ export interface IntentClassifierOptions { export interface IntentClassifyOptions { /** Episode id this classification is being run for, when known. */ episodeId?: EpisodeId; + /** Foreground request cancellation propagated to the provider call. */ + signal?: AbortSignal; } export interface IntentClassifier { @@ -91,7 +93,7 @@ export function createIntentClassifier(opts: IntentClassifierOptions = {}): Inte if (!llmDisabled && llm) { try { const result = await withTimeout( - callLlm(llm, text, options?.episodeId), + callLlm(llm, text, options?.episodeId, timeoutMs, options?.signal), timeoutMs, "intent.llm.timeout", ); @@ -207,6 +209,8 @@ async function callLlm( llm: LlmClient, text: string, episodeId?: EpisodeId, + timeoutMs?: number, + signal?: AbortSignal, ): Promise { const rsp = await llm.completeJson<{ kind: unknown; confidence: unknown; reason: unknown }>( [ @@ -217,6 +221,8 @@ async function callLlm( op: "session.intent.classify", phase: "session", episodeId, + timeoutMs, + signal, schemaHint: `{"kind":"task"|"memory_probe"|"chitchat"|"meta"|"unknown","confidence":0..1,"reason":"..."}`, validate: (v) => { const o = v as Record; diff --git a/apps/memos-local-plugin/core/session/manager.ts b/apps/memos-local-plugin/core/session/manager.ts index 44da570b2..8f0f2e79f 100644 --- a/apps/memos-local-plugin/core/session/manager.ts +++ b/apps/memos-local-plugin/core/session/manager.ts @@ -60,6 +60,8 @@ export interface StartEpisodeInput { /** Adapter-provided event time for the first user turn. */ ts?: EpochMs; meta?: Record; + /** Foreground cancellation propagated to intent classification. */ + signal?: AbortSignal; } export interface SessionManager { @@ -267,6 +269,7 @@ export function createSessionManager(deps: SessionManagerDeps): SessionManager { const episodeId = (input.id ?? ids.episode()) as EpisodeId; const intent = await deps.intentClassifier.classify(input.userMessage, { episodeId, + signal: input.signal, }); // Wrap the write+emit in a log context so downstream listeners inherit diff --git a/apps/memos-local-plugin/core/session/relation-classifier.ts b/apps/memos-local-plugin/core/session/relation-classifier.ts index bac4679e0..f0e1e2a24 100644 --- a/apps/memos-local-plugin/core/session/relation-classifier.ts +++ b/apps/memos-local-plugin/core/session/relation-classifier.ts @@ -306,7 +306,11 @@ export function createRelationClassifier( // Step 2: LLM classification. if (!llmDisabled && opts.llm) { try { - const result = await withTimeout(callLlm(opts.llm, input), timeoutMs, "relation.llm.timeout"); + const result = await withTimeout( + callLlm(opts.llm, input, timeoutMs), + timeoutMs, + "relation.llm.timeout", + ); log.debug("llm.ok", { relation: result.relation, confidence: result.confidence, @@ -326,7 +330,7 @@ export function createRelationClassifier( }); try { const arb = await withTimeout( - callArbitration(opts.llm, input), + callArbitration(opts.llm, input, timeoutMs), timeoutMs, "relation.arbitration.timeout", ); @@ -516,7 +520,11 @@ function buildLlmUserContent(input: RelationInput): string { return parts.join("\n\n"); } -async function callLlm(llm: LlmClient, input: RelationInput): Promise { +async function callLlm( + llm: LlmClient, + input: RelationInput, + timeoutMs?: number, +): Promise { const userContent = buildLlmUserContent(input); const rsp = await llm.completeJson<{ relation: unknown; confidence: unknown; reason: unknown }>( @@ -528,6 +536,8 @@ async function callLlm(llm: LlmClient, input: RelationInput): Promise { const o = v as Record; @@ -581,7 +591,11 @@ When in doubt, choose follow_up. Reply JSON ONLY: {"relation":"follow_up"|"new_task","reason":"..."}`; -async function callArbitration(llm: LlmClient, input: RelationInput): Promise { +async function callArbitration( + llm: LlmClient, + input: RelationInput, + timeoutMs?: number, +): Promise { const userContent = [ `CURRENT TASK CONTEXT:\n${(input.prevUserText ?? "").slice(0, 600)}`, `ASSISTANT REPLY:\n${(input.prevAssistantText ?? "").slice(0, 800)}`, @@ -597,6 +611,8 @@ async function callArbitration(llm: LlmClient, input: RelationInput): Promise { const o = v as Record; diff --git a/apps/memos-local-plugin/core/session/types.ts b/apps/memos-local-plugin/core/session/types.ts index 3d34438ea..62ea53f97 100644 --- a/apps/memos-local-plugin/core/session/types.ts +++ b/apps/memos-local-plugin/core/session/types.ts @@ -193,6 +193,8 @@ export interface RelationInput { * is "scoring whether to terminate prevEpisodeId". */ prevEpisodeId?: EpisodeId; + /** Foreground request cancellation propagated to LLM classification. */ + signal?: AbortSignal; } // ─── Event bus ────────────────────────────────────────────────────────────── diff --git a/apps/memos-local-plugin/core/util/foreground-resources.ts b/apps/memos-local-plugin/core/util/foreground-resources.ts new file mode 100644 index 000000000..d818d189f --- /dev/null +++ b/apps/memos-local-plugin/core/util/foreground-resources.ts @@ -0,0 +1,274 @@ +import type { + EmbedCallOptions, + Embedder, + EmbedInput, +} from "../embedding/types.js"; +import type { EmbeddingVector } from "../types.js"; + +export type ResourcePriority = "foreground" | "background"; + +export interface ForegroundResources { + readonly shutdownSignal: AbortSignal; + /** Combine a request signal with the pipeline lifecycle signal. */ + signalFor(signal?: AbortSignal): AbortSignal; + /** Mark the complete turn.start path as foreground work. Idempotent release. */ + enterForeground(): () => void; + /** Background LLM work waits here before acquiring its existing semaphore. */ + waitForBackground(signal?: AbortSignal): Promise; + /** Priority-aware, non-preemptive embedding admission. */ + acquireEmbedding( + priority: ResourcePriority, + signal?: AbortSignal, + ): Promise<() => void>; + /** Reject queued work and cancel provider calls before pipeline drain. */ + shutdown(reason?: string): void; +} + +export interface ForegroundResourceOptions { + embeddingConcurrency?: number; + /** Prevent background starvation during a sustained foreground stream. */ + maxForegroundBurst?: number; +} + +interface Waiter { + resolve: (release: () => void) => void; + reject: (error: Error) => void; + signal?: AbortSignal; + onAbort?: () => void; +} + +interface BackgroundWaiter { + resolve: () => void; + reject: (error: Error) => void; + signal?: AbortSignal; + onAbort?: () => void; +} + +export function createForegroundResources( + options: ForegroundResourceOptions = {}, +): ForegroundResources { + const capacity = Math.max(1, Math.floor(options.embeddingConcurrency ?? 1)); + const maxForegroundBurst = Math.max( + 1, + Math.floor(options.maxForegroundBurst ?? 8), + ); + const embeddingWaiters: Record = { + foreground: [], + background: [], + }; + const backgroundWaiters: BackgroundWaiter[] = []; + let embeddingInUse = 0; + let foregroundActive = 0; + let foregroundBurst = 0; + const shutdownController = new AbortController(); + + function signalFor(signal?: AbortSignal): AbortSignal { + return signal + ? AbortSignal.any([signal, shutdownController.signal]) + : shutdownController.signal; + } + + function abortError(signal?: AbortSignal): Error { + return signal?.reason instanceof Error + ? signal.reason + : new DOMException("resource wait aborted", "AbortError"); + } + + function removeAbortListener(waiter: Waiter | BackgroundWaiter): void { + if (waiter.signal && waiter.onAbort) { + waiter.signal.removeEventListener("abort", waiter.onAbort); + } + } + + function nextEmbeddingWaiter(): { + priority: ResourcePriority; + waiter: Waiter; + } | null { + const foreground = embeddingWaiters.foreground; + const background = embeddingWaiters.background; + if ( + background.length > 0 && + (foreground.length === 0 || foregroundBurst >= maxForegroundBurst) + ) { + return { priority: "background", waiter: background.shift()! }; + } + if (foreground.length > 0) { + return { priority: "foreground", waiter: foreground.shift()! }; + } + if (background.length > 0) { + return { priority: "background", waiter: background.shift()! }; + } + return null; + } + + function drainEmbedding(): void { + while (embeddingInUse < capacity) { + const next = nextEmbeddingWaiter(); + if (!next) return; + removeAbortListener(next.waiter); + embeddingInUse++; + foregroundBurst = next.priority === "foreground" ? foregroundBurst + 1 : 0; + next.waiter.resolve(makeEmbeddingRelease()); + } + } + + function makeEmbeddingRelease(): () => void { + let released = false; + return (): void => { + if (released) return; + released = true; + embeddingInUse--; + drainEmbedding(); + }; + } + + function acquireEmbedding( + priority: ResourcePriority, + signal?: AbortSignal, + ): Promise<() => void> { + signal = signalFor(signal); + if (signal.aborted) return Promise.reject(abortError(signal)); + return new Promise((resolve, reject) => { + const waiter: Waiter = { resolve, reject, signal }; + if (signal) { + waiter.onAbort = () => { + const queue = embeddingWaiters[priority]; + const index = queue.indexOf(waiter); + if (index >= 0) queue.splice(index, 1); + reject(abortError(signal)); + }; + signal.addEventListener("abort", waiter.onAbort, { once: true }); + } + embeddingWaiters[priority].push(waiter); + drainEmbedding(); + }); + } + + function drainBackgroundGate(): void { + if (foregroundActive > 0) return; + for (const waiter of backgroundWaiters.splice(0)) { + removeAbortListener(waiter); + waiter.resolve(); + } + } + + function enterForeground(): () => void { + foregroundActive++; + let left = false; + return (): void => { + if (left) return; + left = true; + foregroundActive--; + drainBackgroundGate(); + }; + } + + function waitForBackground(signal?: AbortSignal): Promise { + signal = signalFor(signal); + if (signal.aborted) return Promise.reject(abortError(signal)); + if (foregroundActive === 0) return Promise.resolve(); + return new Promise((resolve, reject) => { + const waiter: BackgroundWaiter = { resolve, reject, signal }; + if (signal) { + waiter.onAbort = () => { + const index = backgroundWaiters.indexOf(waiter); + if (index >= 0) backgroundWaiters.splice(index, 1); + reject(abortError(signal)); + }; + signal.addEventListener("abort", waiter.onAbort, { once: true }); + } + backgroundWaiters.push(waiter); + }); + } + + function shutdown(reason = "pipeline shutdown"): void { + if (shutdownController.signal.aborted) return; + shutdownController.abort(new DOMException(reason, "AbortError")); + } + + return { + shutdownSignal: shutdownController.signal, + signalFor, + enterForeground, + waitForBackground, + acquireEmbedding, + shutdown, + }; +} + +/** + * Keep the Embedder contract intact while moving provider round-trips behind + * the shared priority arbiter. Background batches are deliberately chunked + * so one enrichment pass cannot monopolize the provider for an entire queue. + */ +export function prioritizeEmbedder( + inner: Embedder | null, + resources: ForegroundResources, + priority: ResourcePriority, + backgroundChunkSize = 8, +): Embedder | null { + if (!inner) return null; + + async function embedOne( + input: string | EmbedInput, + options?: EmbedCallOptions, + ): Promise { + const signal = resources.signalFor(options?.signal); + const callOptions = { ...options, signal }; + if (priority === "background") await resources.waitForBackground(signal); + const release = await resources.acquireEmbedding(priority, signal); + try { + return await inner!.embedOne(input, callOptions); + } finally { + release(); + } + } + + async function embedMany( + inputs: Array, + options?: EmbedCallOptions, + ): Promise { + const signal = resources.signalFor(options?.signal); + const callOptions = { ...options, signal }; + if (priority === "foreground" || inputs.length <= backgroundChunkSize) { + if (priority === "background") await resources.waitForBackground(signal); + const release = await resources.acquireEmbedding(priority, signal); + try { + return await inner!.embedMany(inputs, callOptions); + } finally { + release(); + } + } + + const results: EmbeddingVector[] = []; + for (let start = 0; start < inputs.length; start += backgroundChunkSize) { + await resources.waitForBackground(signal); + const release = await resources.acquireEmbedding(priority, signal); + try { + results.push( + ...await inner!.embedMany(inputs.slice(start, start + backgroundChunkSize), callOptions), + ); + } finally { + release(); + } + } + return results; + } + + return { + get dimensions() { + return inner.dimensions; + }, + get provider() { + return inner.provider; + }, + get model() { + return inner.model; + }, + embedOne, + embedMany, + stats: () => inner.stats(), + resetCache: () => inner.resetCache(), + close: () => inner.close(), + }; +} diff --git a/apps/memos-local-plugin/core/util/rate-limited-llm.ts b/apps/memos-local-plugin/core/util/rate-limited-llm.ts index 4d229b4c9..5bcb863bb 100644 --- a/apps/memos-local-plugin/core/util/rate-limited-llm.ts +++ b/apps/memos-local-plugin/core/util/rate-limited-llm.ts @@ -10,20 +10,26 @@ import type { LlmStreamChunk, } from "../llm/types.js"; import type { Semaphore } from "./semaphore.js"; +import type { ForegroundResources } from "./foreground-resources.js"; /** * Wrap an LLM client so expensive background subscribers share one * process-wide concurrency budget without changing call-site semantics. */ -export function rateLimitLlmClient(client: LlmClient | null, semaphore: Semaphore): LlmClient | null { +export function rateLimitLlmClient( + client: LlmClient | null, + semaphore: Semaphore, + resources?: ForegroundResources, +): LlmClient | null { if (!client) return null; - return new RateLimitedLlmClient(client, semaphore); + return new RateLimitedLlmClient(client, semaphore, resources); } class RateLimitedLlmClient implements LlmClient { constructor( private readonly inner: LlmClient, private readonly semaphore: Semaphore, + private readonly resources?: ForegroundResources, ) {} get provider(): LlmProviderName { @@ -42,9 +48,12 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCallOptions, ): Promise { - const release = await this.semaphore.acquire(); + const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; + const callOpts = signal ? { ...opts, signal } : opts; + await this.resources?.waitForBackground(signal); + const release = await this.semaphore.acquire(signal); try { - return await this.inner.complete(messages, opts); + return await this.inner.complete(messages, callOpts); } finally { release(); } @@ -54,9 +63,12 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCompleteJsonOptions, ): Promise> { - const release = await this.semaphore.acquire(); + const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; + const callOpts = signal ? { ...opts, signal } : opts; + await this.resources?.waitForBackground(signal); + const release = await this.semaphore.acquire(signal); try { - return await this.inner.completeJson(messages, opts); + return await this.inner.completeJson(messages, callOpts); } finally { release(); } @@ -66,9 +78,12 @@ class RateLimitedLlmClient implements LlmClient { messages: LlmMessage[] | string, opts?: LlmCallOptions, ): AsyncIterable { - const release = await this.semaphore.acquire(); + const signal = this.resources?.signalFor(opts?.signal) ?? opts?.signal; + const callOpts = signal ? { ...opts, signal } : opts; + await this.resources?.waitForBackground(signal); + const release = await this.semaphore.acquire(signal); try { - yield* this.inner.stream(messages, opts); + yield* this.inner.stream(messages, callOpts); } finally { release(); } diff --git a/apps/memos-local-plugin/core/util/request-deadline.ts b/apps/memos-local-plugin/core/util/request-deadline.ts new file mode 100644 index 000000000..a7fb816a0 --- /dev/null +++ b/apps/memos-local-plugin/core/util/request-deadline.ts @@ -0,0 +1,37 @@ +export interface RequestDeadline { + readonly signal: AbortSignal; + remainingMs(): number; + dispose(): void; +} + +/** + * Convert an adapter-provided absolute epoch deadline into one abort signal. + * The absolute form survives JSON-RPC transport time and prevents every stage + * from accidentally receiving a fresh timeout budget. + */ +export function createRequestDeadline( + deadlineAt: number, + now: () => number = Date.now, +): RequestDeadline { + const controller = new AbortController(); + const remainingMs = (): number => Math.max(0, deadlineAt - now()); + const initialRemaining = remainingMs(); + let timer: ReturnType | null = null; + + if (!Number.isFinite(deadlineAt) || initialRemaining <= 0) { + controller.abort(new DOMException("request deadline exceeded", "TimeoutError")); + } else { + timer = setTimeout(() => { + controller.abort(new DOMException("request deadline exceeded", "TimeoutError")); + }, initialRemaining); + } + + return { + signal: controller.signal, + remainingMs, + dispose(): void { + if (timer) clearTimeout(timer); + timer = null; + }, + }; +} diff --git a/apps/memos-local-plugin/core/util/retry-after.ts b/apps/memos-local-plugin/core/util/retry-after.ts new file mode 100644 index 000000000..e580e5f7b --- /dev/null +++ b/apps/memos-local-plugin/core/util/retry-after.ts @@ -0,0 +1,200 @@ +/** Parse RFC 9110 Retry-After delay-seconds or HTTP-date into milliseconds. */ +export const MAX_INLINE_RETRY_DELAY_MS = 30_000; +/** @deprecated Use MAX_INLINE_RETRY_DELAY_MS. */ +export const MAX_RETRY_DELAY_MS = MAX_INLINE_RETRY_DELAY_MS; + +export type RetryDeferReason = + | "deadline_insufficient" + | "retry_after_too_long"; + +export interface RetryPlanBase { + backoffMs: number; + delayMs: number; + retryAfterMs: number | null; + retryAt: number; + source: "backoff" | "retry_after"; +} + +export type RetryPlan = + | (RetryPlanBase & { action: "wait" }) + | (RetryPlanBase & { action: "defer"; reason: RetryDeferReason }); + +export interface RetryCooldown { + retryAfterMs: number; + retryAt: number; + status: number; +} + +export interface RetryDiagnosticDetails { + retryAfterMs?: number; + retryAt?: number; + retryDecision?: "wait" | "defer" | "stop"; + retryReason?: string; +} + +const retryCooldowns = new Map(); + +export function parseRetryAfterMs( + value: string | null | undefined, + nowMs: number = Date.now(), +): number | null { + const raw = value?.trim(); + if (!raw) return null; + if (/^\d+$/.test(raw)) { + const seconds = Number(raw); + const delayMs = seconds * 1_000; + return Number.isSafeInteger(seconds) && Number.isSafeInteger(delayMs) + ? delayMs + : null; + } + // Retry-After only permits IMF-fixdate here. Keeping the shape strict avoids + // JavaScript accepting ambiguous strings such as "1.5" as a legacy date. + if (!/^[A-Za-z]{3}, \d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(raw)) return null; + const at = Date.parse(raw); + if (!Number.isFinite(at)) return null; + return Math.max(0, at - nowMs); +} + +export function retryDelayMs(input: { + attempt: number; + baseMs: number; + jitterMaxMs: number; + retryAfterMs?: number | null; + maxDelayMs?: number; + random?: () => number; +}): number { + const plan = planRetry({ + ...input, + maxInlineDelayMs: input.maxDelayMs, + }); + return plan.delayMs; +} + +/** + * Decide whether a retry can happen inline without violating Retry-After. + * + * Provider Retry-After values are never clamped downward. When the earliest + * legal retry cannot fit the inline wait or request deadline, callers must + * defer/fallback and carry retryAt into their recovery path. + */ +export function planRetry(input: { + attempt: number; + baseMs: number; + jitterMaxMs: number; + retryAfterMs?: number | null; + maxInlineDelayMs?: number; + deadlineAt?: number; + nowMs?: number; + random?: () => number; +}): RetryPlan { + const nowMs = input.nowMs ?? Date.now(); + const random = input.random ?? Math.random; + const jitter = Math.floor(random() * input.jitterMaxMs); + const rawBackoff = input.baseMs * 2 ** Math.max(0, input.attempt - 1) + jitter; + const maxInlineDelayMs = input.maxInlineDelayMs ?? MAX_INLINE_RETRY_DELAY_MS; + const backoffMs = Math.min(rawBackoff, maxInlineDelayMs); + const retryAfterMs = input.retryAfterMs ?? null; + const delayMs = Math.max(backoffMs, retryAfterMs ?? 0); + const retryAt = nowMs + delayMs; + const source = retryAfterMs !== null && retryAfterMs >= backoffMs + ? "retry_after" as const + : "backoff" as const; + const base: RetryPlanBase = { + backoffMs, + delayMs, + retryAfterMs, + retryAt, + source, + }; + + if (retryAfterMs !== null && retryAfterMs > maxInlineDelayMs) { + return { ...base, action: "defer", reason: "retry_after_too_long" }; + } + if (input.deadlineAt !== undefined && retryAt > input.deadlineAt) { + return { ...base, action: "defer", reason: "deadline_insufficient" }; + } + return { ...base, action: "wait" }; +} + +export function retryCooldownKey( + kind: "llm" | "embedding", + provider: string, + url: string, + scope: string = "", +): string { + return `${kind}\u0000${provider}\u0000${url}\u0000${scope}`; +} + +/** Extend a provider cooldown monotonically; a shorter later response cannot weaken it. */ +export function recordRetryCooldown(key: string, cooldown: RetryCooldown): void { + const current = retryCooldowns.get(key); + if (!current || cooldown.retryAt > current.retryAt) { + retryCooldowns.set(key, { ...cooldown }); + } +} + +export function getRetryCooldown( + key: string, + nowMs: number = Date.now(), +): RetryCooldown | null { + const cooldown = retryCooldowns.get(key); + if (!cooldown) return null; + if (cooldown.retryAt <= nowMs) { + retryCooldowns.delete(key); + return null; + } + return { ...cooldown }; +} + +/** Test/runtime-reset hook; plugin shutdown does not need to await cooldown state. */ +export function clearRetryCooldowns(): void { + retryCooldowns.clear(); +} + +/** Copy only bounded, machine-readable retry fields from an error detail bag. */ +export function extractRetryDiagnostics( + details: Record | undefined, +): RetryDiagnosticDetails { + if (!details) return {}; + const diagnostic: RetryDiagnosticDetails = {}; + if (typeof details.retryAfterMs === "number" && Number.isFinite(details.retryAfterMs)) { + diagnostic.retryAfterMs = details.retryAfterMs; + } + if (typeof details.retryAt === "number" && Number.isFinite(details.retryAt)) { + diagnostic.retryAt = details.retryAt; + } + if ( + details.retryDecision === "wait" + || details.retryDecision === "defer" + || details.retryDecision === "stop" + ) { + diagnostic.retryDecision = details.retryDecision; + } + if (typeof details.retryReason === "string") { + diagnostic.retryReason = details.retryReason; + } + return diagnostic; +} + +/** Abortable retry wait so request cancellation and shutdown do not leave sleepers behind. */ +export function waitForRetry(delayMs: number, signal?: AbortSignal): Promise { + if (signal?.aborted) return Promise.reject(abortReason(signal)); + if (delayMs <= 0) return Promise.resolve(); + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, delayMs); + const onAbort = () => { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + reject(signal ? abortReason(signal) : new DOMException("Aborted", "AbortError")); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException("Aborted", "AbortError"); +} diff --git a/apps/memos-local-plugin/core/util/semaphore.ts b/apps/memos-local-plugin/core/util/semaphore.ts index 8dd9b75fc..037db2607 100644 --- a/apps/memos-local-plugin/core/util/semaphore.ts +++ b/apps/memos-local-plugin/core/util/semaphore.ts @@ -1,23 +1,37 @@ export interface Semaphore { - acquire(): Promise<() => void>; + acquire(signal?: AbortSignal): Promise<() => void>; +} + +interface Waiter { + resolve: (release: () => void) => void; + reject: (error: Error) => void; + signal?: AbortSignal; + onAbort?: () => void; } export function createSemaphore(max: number): Semaphore { const limit = Math.max(1, Math.floor(max)); let current = 0; - const waiters: Array<() => void> = []; + const waiters: Waiter[] = []; return { - async acquire() { + async acquire(signal?: AbortSignal) { + if (signal?.aborted) throw abortError(signal); if (current < limit) { current++; return release; } - return new Promise<() => void>((resolve) => { - waiters.push(() => { - current++; - resolve(release); - }); + return new Promise<() => void>((resolve, reject) => { + const waiter: Waiter = { resolve, reject, signal }; + if (signal) { + waiter.onAbort = () => { + const index = waiters.indexOf(waiter); + if (index >= 0) waiters.splice(index, 1); + reject(abortError(signal)); + }; + signal.addEventListener("abort", waiter.onAbort, { once: true }); + } + waiters.push(waiter); }); }, }; @@ -25,6 +39,17 @@ export function createSemaphore(max: number): Semaphore { function release() { current = Math.max(0, current - 1); const next = waiters.shift(); - if (next) next(); + if (!next) return; + if (next.signal && next.onAbort) { + next.signal.removeEventListener("abort", next.onAbort); + } + current++; + next.resolve(release); } } + +function abortError(signal: AbortSignal): Error { + return signal.reason instanceof Error + ? signal.reason + : new DOMException("semaphore wait aborted", "AbortError"); +} diff --git a/apps/memos-local-plugin/tests/python/test_bridge_client.py b/apps/memos-local-plugin/tests/python/test_bridge_client.py index 82763fcea..b5c9eb917 100644 --- a/apps/memos-local-plugin/tests/python/test_bridge_client.py +++ b/apps/memos-local-plugin/tests/python/test_bridge_client.py @@ -448,6 +448,128 @@ def test_reverse_request_waits_for_late_host_handler_registration(self) -> None: self.assertNotIn("error", response) client.close() + def test_slow_reverse_handler_does_not_block_regular_rpc_responses(self) -> None: + """A host LLM callback must not stall the stdout response demux. + + ``host.llm.complete`` can legitimately spend several seconds in the + Hermes model client. The bridge reader still has to resolve an + unrelated foreground ``turn.start`` response during that + window; otherwise one background callback head-of-line blocks every + provider lease sharing the process. + """ + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + handler_started = threading.Event() + release_handler = threading.Event() + + def _slow_handler(_params: dict) -> dict: + handler_started.set() + release_handler.wait(timeout=2.0) + return {"text": "host:done", "model": "host-test"} + + client.register_host_handler("host.llm.complete", _slow_handler) + self._fake.stdout._enqueue( + { + "jsonrpc": "2.0", + "id": "srv-slow", + "method": "host.llm.complete", + "params": {"messages": [{"role": "user", "content": "slow"}]}, + } + ) + self.assertTrue(handler_started.wait(timeout=0.5)) + + try: + response = client.request( + "turn.start", + { + "sessionId": "hermes:session:1", + "userText": "foreground recall", + }, + timeout=0.5, + ) + self.assertIn("foreground recall", response["injectedContext"]) + finally: + release_handler.set() + + reverse_response = self._wait_for_client_write(lambda msg: msg.get("id") == "srv-slow") + self.assertEqual(reverse_response["result"]["text"], "host:done") + client.close() + + def test_reverse_handler_queue_rejects_overload_without_blocking_reader(self) -> None: + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + handler_started = threading.Event() + release_handler = threading.Event() + + def _slow_handler(_params: dict) -> dict: + handler_started.set() + release_handler.wait(timeout=2.0) + return {"text": "done"} + + client.register_host_handler("host.llm.complete", _slow_handler) + self._fake.stdout._enqueue( + { + "jsonrpc": "2.0", + "id": "srv-running", + "method": "host.llm.complete", + "params": {}, + } + ) + self.assertTrue(handler_started.wait(timeout=0.5)) + + overflow_id = "srv-overflow" + for index in range(bridge_client_mod.HOST_HANDLER_QUEUE_CAPACITY + 1): + rpc_id = ( + overflow_id + if index == bridge_client_mod.HOST_HANDLER_QUEUE_CAPACITY + else f"srv-{index}" + ) + self._fake.stdout._enqueue( + { + "jsonrpc": "2.0", + "id": rpc_id, + "method": "host.llm.complete", + "params": {}, + } + ) + + try: + response = self._wait_for_client_write(lambda msg: msg.get("id") == overflow_id) + self.assertEqual(response["error"]["data"]["code"], "host_handler_busy") + finally: + client.close() + release_handler.set() + + def test_close_does_not_wait_for_a_running_reverse_handler(self) -> None: + """An uncooperative host callback must not extend bridge shutdown.""" + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + handler_started = threading.Event() + release_handler = threading.Event() + + def _slow_handler(_params: dict) -> dict: + handler_started.set() + release_handler.wait(timeout=2.0) + return {"text": "late", "model": "host-test"} + + client.register_host_handler("host.llm.complete", _slow_handler) + self._fake.stdout._enqueue( + { + "jsonrpc": "2.0", + "id": "srv-close", + "method": "host.llm.complete", + "params": {}, + } + ) + self.assertTrue(handler_started.wait(timeout=0.5)) + + started = time.monotonic() + try: + client.close() + finally: + release_handler.set() + self.assertLess(time.monotonic() - started, 0.5) + def test_reader_exit_marks_pending_as_transport_closed(self) -> None: """R1 (#2028): reader thread EOF must wake pending waiters with transport_closed instead of leaving them parked on their @@ -1111,7 +1233,7 @@ def test_sync_turn_uses_long_rpc_timeout_for_turn_end(self) -> None: "sessions (issue #2028).", ) - def test_prefetch_uses_long_rpc_timeout_for_turn_start(self) -> None: + def test_prefetch_uses_dedicated_foreground_timeout_for_turn_start(self) -> None: p = self._provider_mod.MemTensorProvider() bridge = RecordingBridge() p._bridge = bridge @@ -1122,12 +1244,45 @@ def test_prefetch_uses_long_rpc_timeout_for_turn_start(self) -> None: self.assertIn("turn.start", methods) start_index = methods.index("turn.start") start_kwargs = bridge.call_kwargs[start_index] - self.assertGreaterEqual( + self.assertGreater(start_kwargs.get("timeout", 0.0), 0.0) + self.assertLessEqual( start_kwargs.get("timeout", 0.0), - self._EXPECTED_LONG_TIMEOUT, - "turn.start suffers the same long-tail latency as turn.end and " - "must share the long RPC timeout (issue #2028).", + self._provider_mod._PREFETCH_RPC_TIMEOUT, + "foreground turn.start must finish before the Hermes host deadline; " + "long capture work keeps the separate issue #2028 timeout.", ) + start_payload = bridge.calls[start_index][1] + self.assertIn("deadlineAt", start_payload) + self.assertGreater(start_payload["deadlineAt"], start_payload["ts"]) + + def test_foreground_reconnect_and_retry_share_one_deadline(self) -> None: + class ClosedBridge: + def request(self, *_args, **_kwargs) -> dict: + raise BridgeError("transport_closed", "bridge closed") + + p = self._provider_mod.MemTensorProvider() + p._bridge = ClosedBridge() + recovered = RecordingBridge() + monotonic_now = [100.0] + + def reconnect(_session_id: str, *, timeout: float) -> None: + self.assertLessEqual(timeout, 6.0) + monotonic_now[0] += 4.0 + p._bridge = recovered + + with ( + patch("memos_provider.time.monotonic", side_effect=lambda: monotonic_now[0]), + patch.object(p, "_reconnect_bridge", side_effect=reconnect), + ): + p._bridge_request_with_retry( + "turn.start", + {"sessionId": "s-1"}, + timeout=6.0, + deadline_monotonic=106.0, + ) + + self.assertEqual(recovered.calls[0][0], "turn.start") + self.assertLessEqual(recovered.call_kwargs[0]["timeout"], 2.0) class ViewerDaemonTests(unittest.TestCase): diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index 6b95ae61b..ecd0e1ae1 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -480,6 +480,75 @@ def test_prefetch_passes_stable_turn_key_to_bridge(self) -> None: turn_start = next(params for method, params in bridge.calls if method == "turn.start") self.assertEqual(turn_start["turnKey"], "turn-key-session:7") + def test_prefetch_uses_a_dedicated_budget_and_forwards_absolute_deadline(self) -> None: + bridge = FakeBridge() + with ( + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.MemosBridgeClient", return_value=bridge), + patch("memos_provider._PREFETCH_RPC_TIMEOUT", 6.0), + patch("memos_provider.time.time", return_value=1_700_000_000.0), + ): + provider = memos_provider.MemTensorProvider() + provider.initialize("budget-session") + provider.on_turn_start(1, "recall the build decision") + with patch.object( + provider, + "_bridge_request_with_retry", + wraps=provider._bridge_request_with_retry, + ) as request: + provider.prefetch("recall the build decision") + + turn_start = next(params for method, params in bridge.calls if method == "turn.start") + self.assertEqual(turn_start["deadlineAt"], 1_700_000_005_750) + request.assert_called_once() + self.assertLessEqual(request.call_args.kwargs["timeout"], 6.0) + self.assertIn("deadline_monotonic", request.call_args.kwargs) + + def test_prefetch_budget_includes_bridge_ensure_time(self) -> None: + bridge = FakeBridge() + monotonic_now = [100.0] + + def ensure_bridge(_session_id: str, *, timeout: float) -> bool: + self.assertAlmostEqual(timeout, 6.0, places=3) + monotonic_now[0] += 2.5 + return True + + with ( + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.MemosBridgeClient", return_value=bridge), + patch("memos_provider._PREFETCH_RPC_TIMEOUT", 6.0), + patch("memos_provider.time.time", return_value=1_700_000_000.0), + patch("memos_provider.time.monotonic", side_effect=lambda: monotonic_now[0]), + ): + provider = memos_provider.MemTensorProvider() + provider.initialize("budget-session") + provider.on_turn_start(1, "recall the build decision") + with ( + patch.object(provider, "_ensure_bridge", side_effect=ensure_bridge), + patch.object( + provider, + "_bridge_request_with_retry", + wraps=provider._bridge_request_with_retry, + ) as request, + ): + provider.prefetch("recall the build decision") + + self.assertLessEqual(request.call_args.kwargs["timeout"], 3.5) + turn_start = next(params for method, params in bridge.calls if method == "turn.start") + self.assertEqual(turn_start["deadlineAt"], 1_700_000_005_750) + + def test_prefetch_timeout_config_rejects_non_positive_values(self) -> None: + with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "0"}): + self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 6.0) + with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "nan"}): + self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 6.0) + with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "4.5"}): + self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 4.5) + with patch.dict("os.environ", {"MEMOS_HERMES_PREFETCH_RPC_TIMEOUT": "30"}): + self.assertEqual(memos_provider._prefetch_rpc_timeout_default(), 7.0) + def test_prefetch_suppresses_memory_injection_for_explicit_delegation(self) -> None: bridge = FakeBridge() with ( diff --git a/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts b/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts index 8ee5fa921..fc9f921e1 100644 --- a/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts +++ b/apps/memos-local-plugin/tests/unit/bridge/methods.test.ts @@ -344,6 +344,17 @@ describe("makeDispatcher", () => { ).rejects.toSatisfy( (err) => err instanceof MemosError && err.code === "invalid_argument", ); + await expect( + dispatch("turn.start", { + agent: "openclaw", + sessionId: "s-1", + userText: "hi", + ts: 123, + deadlineAt: "soon", + }), + ).rejects.toSatisfy( + (err) => err instanceof MemosError && err.code === "invalid_argument", + ); }); it("feedback.submit forwards the DTO shape intact", async () => { diff --git a/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts b/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts index 3c608a53a..617bc4a72 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/embedder.test.ts @@ -6,8 +6,10 @@ import { initTestLogger } from "../../../core/logger/index.js"; import type { EmbedRole, EmbeddingConfig, + EmbeddingErrorDetail, EmbeddingProvider, EmbeddingProviderName, + EmbeddingStatusDetail, ProviderCallCtx, } from "../../../core/embedding/types.js"; @@ -75,6 +77,24 @@ describe("embedder facade", () => { expect(Array.from(v)).toEqual([3, 97, 0]); // a=97 }); + it("forwards the caller abort signal and deadline to the provider", async () => { + const seen: Array> = []; + const p: EmbeddingProvider = { + name: "openai_compatible", + async embed(texts, _role, ctx) { + seen.push({ signal: ctx.signal, deadlineAt: ctx.deadlineAt }); + return texts.map(() => [1, 2, 3]); + }, + }; + const e = createEmbedderWithProvider(cfg(), p); + const controller = new AbortController(); + + const deadlineAt = Date.now() + 1_000; + await e.embedOne("signal", { signal: controller.signal, deadlineAt }); + + expect(seen).toEqual([{ signal: controller.signal, deadlineAt }]); + }); + it("dedups identical inputs into one provider call", async () => { const p = new FakeProvider(); const e = createEmbedderWithProvider(cfg(), p); @@ -175,6 +195,45 @@ describe("embedder facade", () => { } }); + it("preserves deferred retry diagnostics in error and status sinks", async () => { + const errors: EmbeddingErrorDetail[] = []; + const statuses: EmbeddingStatusDetail[] = []; + const retryAt = Date.now() + 120_000; + const provider: EmbeddingProvider = { + name: "openai_compatible", + async embed() { + throw new MemosError("embedding_unavailable", "provider cooldown", { + retryAfterMs: 120_000, + retryAt, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }); + }, + }; + const e = createEmbedderWithProvider( + cfg({ onError: (detail) => errors.push(detail), onStatus: (detail) => statuses.push(detail) }), + provider, + ); + + await expect(e.embedOne("x")).rejects.toBeInstanceOf(MemosError); + + expect(errors).toContainEqual( + expect.objectContaining({ + retryAfterMs: 120_000, + retryAt, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }), + ); + expect(statuses).toContainEqual( + expect.objectContaining({ + status: "error", + retryAt, + retryDecision: "defer", + }), + ); + }); + it("rejects when provider returns too few rows", async () => { const e = createEmbedderWithProvider(cfg({ provider: "gemini" }), new WrongCountProvider()); await expect(e.embedMany(["x", "y", "z"])).rejects.toBeInstanceOf(MemosError); diff --git a/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts b/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts index 3b9ee5bf2..2811e94ef 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/fetcher.test.ts @@ -4,6 +4,7 @@ import { MemosError } from "../../../agent-contract/errors.js"; import { initTestLogger } from "../../../core/logger/index.js"; import { httpPostJson } from "../../../core/embedding/fetcher.js"; import type { ProviderLogger } from "../../../core/embedding/types.js"; +import { clearRetryCooldowns } from "../../../core/util/retry-after.js"; function nullLogger(): ProviderLogger { return { @@ -21,6 +22,8 @@ describe("embedding/fetcher", () => { vi.useRealTimers(); // retry backoff uses real setTimeout; keep it real but short }); afterEach(() => { + clearRetryCooldowns(); + vi.useRealTimers(); vi.unstubAllGlobals(); }); @@ -81,6 +84,85 @@ describe("embedding/fetcher", () => { expect(f).toHaveBeenCalledTimes(2); }); + it("honors Retry-After HTTP-date before retrying a 429", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + const f = mockFetch([ + new Response("rate limited", { + status: 429, + headers: { "Retry-After": "Tue, 04 Aug 2026 00:00:03 GMT" }, + }), + new Response(JSON.stringify({ ok: 1 }), { status: 200 }), + ]); + + const pending = httpPostJson<{ ok: number }>({ + url: "https://x", + body: {}, + provider: "cohere", + log: nullLogger(), + maxRetries: 1, + }); + await vi.advanceTimersByTimeAsync(2_999); + expect(f).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + await expect(pending).resolves.toEqual({ ok: 1 }); + expect(f).toHaveBeenCalledTimes(2); + vi.useRealTimers(); + }); + + it("defers a long Retry-After and short-circuits the provider cooldown", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + const f = mockFetch([ + new Response("maintenance", { status: 503, headers: { "Retry-After": "120" } }), + ]); + const opts = { + url: "https://embedding-x", + body: {}, + provider: "cohere" as const, + log: nullLogger(), + maxRetries: 1, + }; + + await expect(httpPostJson(opts)).rejects.toMatchObject({ + code: "embedding_unavailable", + details: { + retryAfterMs: 120_000, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }, + }); + await expect(httpPostJson(opts)).rejects.toMatchObject({ + code: "embedding_unavailable", + details: { retryReason: "cooldown_active" }, + }); + expect(f).toHaveBeenCalledTimes(1); + }); + + it("returns structured diagnostics when network backoff cannot fit the deadline", async () => { + vi.useFakeTimers(); + const now = Date.parse("2026-08-04T00:00:00.000Z"); + vi.setSystemTime(now); + const f = mockFetch([new Error("ECONNRESET")]); + + await expect(httpPostJson({ + url: "https://embedding-deadline", + body: {}, + provider: "mistral", + log: nullLogger(), + maxRetries: 1, + deadlineAt: now + 100, + })).rejects.toMatchObject({ + name: "MemosError", + code: "embedding_unavailable", + details: { + retryDecision: "defer", + retryReason: "deadline_insufficient", + }, + }); + expect(f).toHaveBeenCalledTimes(1); + }); + it("does not retry on 400", async () => { mockFetch([new Response("bad", { status: 400 })]); await expect( diff --git a/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts b/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts index f08a37f32..ae72cc60e 100644 --- a/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts +++ b/apps/memos-local-plugin/tests/unit/embedding/retry-worker.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { createEmbeddingRetryWorker } from "../../../core/embedding/retry-worker.js"; +import { ERROR_CODES, MemosError } from "../../../agent-contract/errors.js"; import { rootLogger } from "../../../core/logger/index.js"; import type { EpisodeId, SessionId, TraceId } from "../../../core/types.js"; import { makeTmpDb, type TmpDbHandle } from "../../helpers/tmp-db.js"; @@ -174,6 +175,39 @@ describe("embedding retry worker", () => { expect(handle.repos.apiLogs.list({ toolName: "system_error", limit: 5, offset: 0 })).toHaveLength(1); }); + it("never schedules a durable retry before the provider retryAt", async () => { + const retryAt = NOW + 120_000; + handle.repos.embeddingRetryQueue.enqueue({ + id: "er_retry_after", + targetKind: "trace", + targetId: "tr_retry", + vectorField: "vec_summary", + sourceText: "retry me later", + maxAttempts: 3, + now: NOW, + }); + const worker = createEmbeddingRetryWorker({ + repos: handle.repos, + embedder: fakeEmbedder({ + throwWith: new MemosError( + ERROR_CODES.EMBEDDING_UNAVAILABLE, + "provider cooling down", + { retryAt, retryAfterMs: 120_000, retryDecision: "defer" }, + ), + }), + log: rootLogger.child({ channel: "test.embedding.retry" }), + now: () => NOW, + }); + + await worker.flush(); + + expect(queueRow(handle, "er_retry_after")).toMatchObject({ + status: "pending", + attempts: 1, + next_attempt_at: retryAt, + }); + }); + it("treats missing target rows as retry failures", async () => { handle.repos.embeddingRetryQueue.enqueue({ id: "er_missing", @@ -201,4 +235,30 @@ describe("embedding retry worker", () => { last_error: "embedding retry target not found: trace:tr_missing", }); }); + + it("does not claim new retry jobs after stop during shutdown", async () => { + handle.repos.embeddingRetryQueue.enqueue({ + id: "er_shutdown", + targetKind: "trace", + targetId: "tr_retry", + vectorField: "vec_summary", + sourceText: "do not start during shutdown", + now: NOW, + }); + const worker = createEmbeddingRetryWorker({ + repos: handle.repos, + embedder: fakeEmbedder({ dimensions: 8 }), + log: rootLogger.child({ channel: "test.embedding.retry" }), + now: () => NOW, + }); + + worker.stop(); + await worker.flush(); + + expect(queueRow(handle, "er_shutdown")).toMatchObject({ + status: "pending", + attempts: 0, + claimed_by: null, + }); + }); }); diff --git a/apps/memos-local-plugin/tests/unit/llm/client.test.ts b/apps/memos-local-plugin/tests/unit/llm/client.test.ts index dee0de228..cd125ecd8 100644 --- a/apps/memos-local-plugin/tests/unit/llm/client.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/client.test.ts @@ -283,6 +283,45 @@ describe("llm/client", () => { await expect(client.complete([] as LlmMessage[])).rejects.toBeInstanceOf(MemosError); }); + it("preserves deferred retry diagnostics in error and status sinks", async () => { + const errors: Array> = []; + const statuses: LlmStatusDetail[] = []; + const retryAt = Date.now() + 120_000; + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_RATE_LIMITED, "provider cooldown", { + retryAfterMs: 120_000, + retryAt, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }), + ); + const client = createLlmClientWithProvider( + cfg({ + onError: (detail) => errors.push(detail as unknown as Record), + onStatus: (detail) => statuses.push(detail), + }), + provider, + ); + + await expect(client.complete("x")).rejects.toBeInstanceOf(MemosError); + + expect(errors).toContainEqual( + expect.objectContaining({ + retryAfterMs: 120_000, + retryAt, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }), + ); + expect(statuses).toContainEqual( + expect.objectContaining({ + status: "error", + retryAt, + retryDecision: "defer", + }), + ); + }); + // ─── Circuit breaker (issue #1897) ────────────────────────────────────── describe("circuit breaker", () => { function statusSink(): { rows: LlmStatusDetail[]; push: (d: LlmStatusDetail) => void } { diff --git a/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts b/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts index a3941b7aa..9b17e1b16 100644 --- a/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/fetcher.test.ts @@ -4,6 +4,7 @@ import { MemosError } from "../../../agent-contract/errors.js"; import { decodeSse, httpPostJson, httpPostStream } from "../../../core/llm/fetcher.js"; import { initTestLogger } from "../../../core/logger/index.js"; import type { LlmProviderLogger } from "../../../core/llm/types.js"; +import { clearRetryCooldowns } from "../../../core/util/retry-after.js"; function nullLog(): LlmProviderLogger { return { @@ -29,7 +30,205 @@ function mockFetch(replies: Array) { describe("llm/fetcher", () => { beforeAll(() => initTestLogger()); - afterEach(() => vi.unstubAllGlobals()); + afterEach(() => { + clearRetryCooldowns(); + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + it("honors Retry-After delay-seconds before retrying a 429", async () => { + vi.useFakeTimers(); + const f = mockFetch([ + new Response("slow down", { status: 429, headers: { "Retry-After": "2" } }), + new Response(JSON.stringify({ ok: 1 }), { status: 200 }), + ]); + + const pending = httpPostJson({ + url: "https://x", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + provider: "openai_compatible", + log: nullLog(), + }); + await vi.advanceTimersByTimeAsync(1_999); + expect(f).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + await expect(pending).resolves.toMatchObject({ json: { ok: 1 } }); + expect(f).toHaveBeenCalledTimes(2); + vi.useRealTimers(); + }); + + it("aborts while waiting for Retry-After", async () => { + vi.useFakeTimers(); + const ctrl = new AbortController(); + const f = mockFetch([ + new Response("slow down", { status: 429, headers: { "Retry-After": "2" } }), + ]); + + const pending = httpPostJson({ + url: "https://x", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + signal: ctrl.signal, + provider: "openai_compatible", + log: nullLog(), + }); + await vi.advanceTimersByTimeAsync(0); + ctrl.abort(); + await expect(pending).rejects.toBeInstanceOf(MemosError); + expect(f).toHaveBeenCalledTimes(1); + vi.useRealTimers(); + }); + + it("defers a long Retry-After from a 503 without retrying early", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + const warn = vi.fn(); + const f = mockFetch([ + new Response("maintenance", { status: 503, headers: { "Retry-After": "120" } }), + ]); + + const pending = httpPostJson({ + url: "https://x", + body: {}, + timeoutMs: 120_000, + maxRetries: 1, + provider: "openai_compatible", + log: { ...nullLog(), warn }, + }); + await expect(pending).rejects.toMatchObject({ + code: "llm_unavailable", + details: { + retryAfterMs: 120_000, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }, + }); + expect(f).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith( + "http.retry_deferred", + expect.objectContaining({ + retryAfterMs: 120_000, + retryDecision: "defer", + retryReason: "retry_after_too_long", + }), + ); + }); + + it("short-circuits calls while the provider Retry-After cooldown is active", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + const warn = vi.fn(); + const f = mockFetch([ + new Response("slow down", { status: 429, headers: { "Retry-After": "120" } }), + ]); + const opts = { + url: "https://x", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + provider: "openai_compatible" as const, + log: { ...nullLog(), warn }, + }; + + await expect(httpPostJson(opts)).rejects.toMatchObject({ code: "llm_rate_limited" }); + await expect(httpPostJson(opts)).rejects.toMatchObject({ + code: "llm_rate_limited", + details: { retryDecision: "defer", retryReason: "cooldown_active" }, + }); + expect(f).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith( + "http.retry_cooldown", + expect.objectContaining({ retryReason: "cooldown_active" }), + ); + }); + + it("does not enter a Retry-After wait that cannot fit the absolute deadline", async () => { + vi.useFakeTimers(); + const now = Date.parse("2026-08-04T00:00:00.000Z"); + vi.setSystemTime(now); + const f = mockFetch([ + new Response("slow down", { status: 429, headers: { "Retry-After": "5" } }), + ]); + + await expect(httpPostJson({ + url: "https://deadline", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + deadlineAt: now + 1_000, + provider: "openai_compatible", + log: nullLog(), + })).rejects.toMatchObject({ + code: "llm_rate_limited", + details: { + retryDecision: "defer", + retryReason: "deadline_insufficient", + }, + }); + expect(f).toHaveBeenCalledTimes(1); + }); + + it("returns structured diagnostics when network backoff cannot fit the deadline", async () => { + vi.useFakeTimers(); + const now = Date.parse("2026-08-04T00:00:00.000Z"); + vi.setSystemTime(now); + const f = mockFetch([new Error("ECONNRESET")]); + + await expect(httpPostJson({ + url: "https://network-deadline", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + deadlineAt: now + 100, + provider: "openai_compatible", + log: nullLog(), + })).rejects.toMatchObject({ + name: "MemosError", + code: "llm_unavailable", + details: { + retryDecision: "defer", + retryReason: "deadline_insufficient", + }, + }); + expect(f).toHaveBeenCalledTimes(1); + }); + + it("does not let an older in-flight success clear a newer provider cooldown", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-04T00:00:00.000Z")); + let resolveSuccess!: (response: Response) => void; + const success = new Promise((resolve) => { resolveSuccess = resolve; }); + const f = vi.fn() + .mockImplementationOnce(() => success) + .mockResolvedValueOnce( + new Response("slow down", { status: 429, headers: { "Retry-After": "120" } }), + ); + vi.stubGlobal("fetch", f); + const opts = { + url: "https://shared-endpoint", + body: {}, + timeoutMs: 5_000, + maxRetries: 1, + provider: "openai_compatible" as const, + log: nullLog(), + }; + + const older = httpPostJson<{ ok: boolean }>(opts); + await vi.waitFor(() => expect(f).toHaveBeenCalledTimes(1)); + await expect(httpPostJson(opts)).rejects.toMatchObject({ + details: { retryReason: "retry_after_too_long" }, + }); + resolveSuccess(new Response(JSON.stringify({ ok: true }), { status: 200 })); + await expect(older).resolves.toMatchObject({ json: { ok: true } }); + + await expect(httpPostJson(opts)).rejects.toMatchObject({ + details: { retryReason: "cooldown_active" }, + }); + expect(f).toHaveBeenCalledTimes(2); + }); it("returns parsed JSON on 200", async () => { mockFetch([new Response(JSON.stringify({ a: 1 }), { status: 200 })]); diff --git a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts index cb5140d09..52853ff4a 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts @@ -898,6 +898,53 @@ describe("MemoryCore façade", () => { expect(scored.priority).toBe(1); }); + it("logs both user and assistant content with the matching role", async () => { + pipeline = createPipeline(buildDeps(db!)); + core = createMemoryCore( + pipeline, + resolveHome("openclaw", "/tmp/memos-mc-test"), + "test", + ); + await core.init(); + + const userText = "今晚吃什么，推荐一下"; + const agentText = "可以考虑清淡的汤面、盖饭或者附近评价不错的家常菜。"; + const start = await core.onTurnStart({ + agent: "openclaw", + sessionId: "s-memory-add-roles", + userText, + ts: 1_700_000_000_000, + }); + await core.onTurnEnd({ + agent: "openclaw", + sessionId: start.query.sessionId!, + episodeId: start.query.episodeId!, + agentText, + toolCalls: [], + ts: 1_700_000_000_500, + }); + + const { logs } = await core.listApiLogs({ + toolName: "memory_add", + limit: 10, + }); + const liteLog = logs.find((log) => { + const input = JSON.parse(log.inputJson) as { phase?: string }; + return input.phase === "lite"; + }); + expect(liteLog).toBeDefined(); + + const output = JSON.parse(liteLog!.outputJson) as { + details?: Array<{ role?: string; content?: string }>; + }; + expect( + output.details?.map(({ role, content }) => ({ role, content })), + ).toEqual([ + { role: "user", content: userText }, + { role: "assistant", content: agentText }, + ]); + }); + it("onTurnEnd preserves adapter-provided historical timestamps", async () => { pipeline = createPipeline(buildDeps(db!)); core = createMemoryCore( diff --git a/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts index a70fdcf6b..1e79c861a 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/orchestrator.test.ts @@ -77,6 +77,39 @@ afterEach(async () => { }); describe("pipeline/orchestrator", () => { + it("degrades retrieval at the adapter deadline and aborts the provider call", async () => { + const base = fakeEmbedder({ dimensions: 384 }); + let sawAbort = false; + const embedder = { + ...base, + async embedOne(input: Parameters[0], options?: { signal?: AbortSignal }) { + return await new Promise>>((resolve, reject) => { + const onAbort = () => { + sawAbort = true; + reject(new DOMException("deadline", "AbortError")); + }; + if (options?.signal?.aborted) return onAbort(); + options?.signal?.addEventListener("abort", onAbort, { once: true }); + void resolve; + }); + }, + }; + pipeline = createPipeline(buildDeps(dbHandle!, embedder)); + const startedAt = Date.now(); + + const packet = await pipeline.onTurnStart({ + agent: "hermes", + sessionId: "s-deadline", + userText: "find the previous build decision", + ts: Date.now(), + deadlineAt: Date.now() + 25, + }); + + expect(sawAbort).toBe(true); + expect(Date.now() - startedAt).toBeLessThan(500); + expect(packet.reason).toBe("turn_start"); + }); + it("threads a dedicated l3Llm through to the handle", () => { const l3Llm = fakeLlm({ completeJson: {} }); pipeline = createPipeline({ ...buildDeps(dbHandle!), l3Llm }); diff --git a/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts b/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts index 6b37ae55f..c3321635a 100644 --- a/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts +++ b/apps/memos-local-plugin/tests/unit/session/intent-classifier.test.ts @@ -109,6 +109,25 @@ describe("session/intent-classifier", () => { expect(d.signals).toEqual(["llm"]); }); + it("forwards the foreground abort signal and classifier timeout", async () => { + let seen: { signal?: AbortSignal; timeoutMs?: number } | undefined; + const llm = fakeLlm(() => ({ kind: "task", confidence: 0.8, reason: "task" })); + const original = llm.completeJson.bind(llm); + llm.completeJson = async (messages, opts) => { + seen = opts; + return original(messages, opts); + }; + const controller = new AbortController(); + const c = createIntentClassifier({ llm, timeoutMs: 321 }); + + await c.classify("investigate an ambiguous pipeline issue", { + signal: controller.signal, + }); + + expect(seen?.signal).toBe(controller.signal); + expect(seen?.timeoutMs).toBe(321); + }); + it("LLM failure falls back to heuristic", async () => { const c = createIntentClassifier({ llm: fakeLlm(() => { diff --git a/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts b/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts index 575a4b44a..a17d3d9ce 100644 --- a/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts +++ b/apps/memos-local-plugin/tests/unit/session/relation-classifier.test.ts @@ -119,6 +119,33 @@ describe("relation-classifier — V7 §0.1", () => { expect(d.llmModel).toBe("fake/test-model"); }); + it("forwards the foreground abort signal and classifier timeout", async () => { + let seen: { signal?: AbortSignal; timeoutMs?: number } | undefined; + const controller = new AbortController(); + const c = createRelationClassifier({ + timeoutMs: 456, + llm: { + completeJson: async (_messages, opts) => { + seen = opts; + return { + value: { relation: "follow_up", confidence: 0.9, reason: "same task" }, + servedBy: "fake/llm", + } as never; + }, + } as LlmClient, + }); + + await c.classify({ + prevUserText: "investigate retrieval latency", + prevAssistantText: "I found several possible causes.", + newUserText: "could the queue contribute to this behavior?", + signal: controller.signal, + }); + + expect(seen?.signal).toBe(controller.signal); + expect(seen?.timeoutMs).toBe(456); + }); + it("falls back to heuristic when LLM throws", async () => { const llm: Partial = { completeJson: async () => { diff --git a/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts b/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts new file mode 100644 index 000000000..1380b9a47 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; + +import { + createForegroundResources, + prioritizeEmbedder, +} from "../../../core/util/foreground-resources.js"; +import { fakeEmbedder } from "../../helpers/fake-embedder.js"; + +describe("foreground resources", () => { + it("admits a queued foreground embedding before queued background work", async () => { + const resources = createForegroundResources({ embeddingConcurrency: 1 }); + const first = await resources.acquireEmbedding("background"); + const order: string[] = []; + + const background = resources.acquireEmbedding("background").then((release) => { + order.push("background"); + release(); + }); + const foreground = resources.acquireEmbedding("foreground").then((release) => { + order.push("foreground"); + release(); + }); + + first(); + await Promise.all([foreground, background]); + + expect(order).toEqual(["foreground", "background"]); + }); + + it("lets background work progress after a bounded foreground burst", async () => { + const resources = createForegroundResources({ + embeddingConcurrency: 1, + maxForegroundBurst: 2, + }); + const first = await resources.acquireEmbedding("foreground"); + const order: string[] = []; + + const background = resources.acquireEmbedding("background").then((release) => { + order.push("background"); + release(); + }); + const foreground1 = resources.acquireEmbedding("foreground").then((release) => { + order.push("foreground-1"); + release(); + }); + const foreground2 = resources.acquireEmbedding("foreground").then((release) => { + order.push("foreground-2"); + release(); + }); + + first(); + await Promise.all([background, foreground1, foreground2]); + + expect(order).toEqual(["foreground-1", "background", "foreground-2"]); + }); + + it("does not start background work while a foreground turn is active", async () => { + const resources = createForegroundResources(); + const leaveForeground = resources.enterForeground(); + let started = false; + + const waiting = resources.waitForBackground().then(() => { + started = true; + }); + await Promise.resolve(); + expect(started).toBe(false); + + leaveForeground(); + await waiting; + expect(started).toBe(true); + }); + + it("removes an aborted embedding waiter without consuming capacity", async () => { + const resources = createForegroundResources({ embeddingConcurrency: 1 }); + const first = await resources.acquireEmbedding("background"); + const controller = new AbortController(); + const waiting = resources.acquireEmbedding("foreground", controller.signal); + + controller.abort(); + await expect(waiting).rejects.toMatchObject({ name: "AbortError" }); + first(); + + const release = await resources.acquireEmbedding("background"); + release(); + }); + + it("chunks background embedding batches and yields between chunks", async () => { + const resources = createForegroundResources({ embeddingConcurrency: 1 }); + const base = fakeEmbedder({ dimensions: 4 }); + const batchSizes: number[] = []; + const inner = { + ...base, + async embedMany(...args: Parameters) { + batchSizes.push(args[0].length); + return base.embedMany(...args); + }, + }; + const background = prioritizeEmbedder(inner, resources, "background", 2)!; + + await background.embedMany(["a", "b", "c", "d", "e"]); + + expect(batchSizes).toEqual([2, 2, 1]); + }); + + it("aborts queued and in-flight provider work during shutdown", async () => { + const resources = createForegroundResources({ embeddingConcurrency: 1 }); + const base = fakeEmbedder({ dimensions: 4 }); + let providerSignal: AbortSignal | undefined; + const inner = { + ...base, + async embedOne( + _input: Parameters[0], + options?: Parameters[1], + ) { + providerSignal = options?.signal; + return await new Promise((_resolve, reject) => { + if (options?.signal?.aborted) { + reject(options.signal.reason); + return; + } + options?.signal?.addEventListener( + "abort", + () => reject(options.signal?.reason), + { once: true }, + ); + }); + }, + }; + const background = prioritizeEmbedder(inner, resources, "background")!; + const pending = background.embedOne("slow background work"); + await Promise.resolve(); + + resources.shutdown("test shutdown"); + + await expect(pending).rejects.toMatchObject({ name: "AbortError" }); + expect(providerSignal?.aborted).toBe(true); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts b/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts new file mode 100644 index 000000000..8d6b8d5a2 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/util/request-deadline.test.ts @@ -0,0 +1,35 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { createRequestDeadline } from "../../../core/util/request-deadline.js"; + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("createRequestDeadline", () => { + it("aborts at the absolute deadline and reports no remaining budget", async () => { + vi.useFakeTimers(); + vi.setSystemTime(1_000); + + const deadline = createRequestDeadline(1_250); + expect(deadline.remainingMs()).toBe(250); + expect(deadline.signal.aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(250); + + expect(deadline.signal.aborted).toBe(true); + expect(deadline.remainingMs()).toBe(0); + deadline.dispose(); + }); + + it("treats an already-expired deadline as immediately aborted", () => { + vi.useFakeTimers(); + vi.setSystemTime(2_000); + + const deadline = createRequestDeadline(1_999); + + expect(deadline.signal.aborted).toBe(true); + expect(deadline.remainingMs()).toBe(0); + deadline.dispose(); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts b/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts new file mode 100644 index 000000000..84a4c8ea4 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/util/retry-after.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; + +import { + clearRetryCooldowns, + getRetryCooldown, + parseRetryAfterMs, + planRetry, + recordRetryCooldown, + retryCooldownKey, +} from "../../../core/util/retry-after.js"; + +describe("parseRetryAfterMs", () => { + it("parses delay-seconds", () => { + expect(parseRetryAfterMs("3", 1_000)).toBe(3_000); + expect(parseRetryAfterMs("0", 1_000)).toBe(0); + }); + + it("parses an HTTP-date relative to the supplied clock", () => { + const now = Date.parse("2026-08-04T00:00:00.000Z"); + expect(parseRetryAfterMs("Tue, 04 Aug 2026 00:00:05 GMT", now)).toBe(5_000); + }); + + it("clamps past HTTP-dates and rejects malformed values", () => { + const now = Date.parse("2026-08-04T00:00:00.000Z"); + expect(parseRetryAfterMs("Mon, 03 Aug 2026 23:59:59 GMT", now)).toBe(0); + expect(parseRetryAfterMs("1.5", now)).toBeNull(); + expect(parseRetryAfterMs("9007199254740991", now)).toBeNull(); + expect(parseRetryAfterMs("later", now)).toBeNull(); + expect(parseRetryAfterMs(null, now)).toBeNull(); + }); + + it("defers instead of retrying before a long provider Retry-After", () => { + expect(planRetry({ + attempt: 1, + baseMs: 200, + jitterMaxMs: 0, + retryAfterMs: 120_000, + maxInlineDelayMs: 30_000, + nowMs: 1_000, + })).toEqual({ + action: "defer", + backoffMs: 200, + delayMs: 120_000, + reason: "retry_after_too_long", + retryAfterMs: 120_000, + retryAt: 121_000, + source: "retry_after", + }); + }); + + it("defers when an otherwise short retry cannot fit the request deadline", () => { + expect(planRetry({ + attempt: 1, + baseMs: 200, + jitterMaxMs: 0, + retryAfterMs: 2_000, + deadlineAt: 2_500, + nowMs: 1_000, + })).toMatchObject({ + action: "defer", + reason: "deadline_insufficient", + retryAt: 3_000, + }); + }); + + it("keeps provider cooldowns monotonic and expires them at retryAt", () => { + clearRetryCooldowns(); + recordRetryCooldown("llm:test", { + retryAfterMs: 2_000, + retryAt: 3_000, + status: 429, + }); + recordRetryCooldown("llm:test", { + retryAfterMs: 500, + retryAt: 1_500, + status: 503, + }); + expect(getRetryCooldown("llm:test", 2_999)).toMatchObject({ + retryAt: 3_000, + status: 429, + }); + expect(getRetryCooldown("llm:test", 3_000)).toBeNull(); + clearRetryCooldowns(); + }); + + it("scopes provider cooldowns by endpoint and model", () => { + expect(retryCooldownKey("llm", "openai_compatible", "https://x", "model-a")) + .not.toBe(retryCooldownKey("llm", "openai_compatible", "https://x", "model-b")); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts b/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts new file mode 100644 index 000000000..2a5308ad2 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/util/semaphore.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from "vitest"; + +import { createSemaphore } from "../../../core/util/semaphore.js"; + +describe("semaphore", () => { + it("removes an aborted waiter so shutdown cannot hang behind active work", async () => { + const semaphore = createSemaphore(1); + const release = await semaphore.acquire(); + const controller = new AbortController(); + const waiting = semaphore.acquire(controller.signal); + + controller.abort(); + await expect(waiting).rejects.toMatchObject({ name: "AbortError" }); + release(); + + const next = await semaphore.acquire(); + next(); + }); +}); From 0dad4af3b7e812dfab57e16ee8ebe2cf3aeaab1b Mon Sep 17 00:00:00 2001 From: jiachengzhen Date: Fri, 7 Aug 2026 00:16:23 +0800 Subject: [PATCH 10/34] fix(plugin): archive idle skills atomically --- .../core/skill/subscriber.ts | 11 +++- .../core/storage/repos/skills.ts | 26 +++++++++ .../tests/unit/skill/subscriber.test.ts | 58 +++++++++++++++++++ .../tests/unit/storage/repos.test.ts | 12 ++++ 4 files changed, 105 insertions(+), 2 deletions(-) diff --git a/apps/memos-local-plugin/core/skill/subscriber.ts b/apps/memos-local-plugin/core/skill/subscriber.ts index 9dd861a73..2d2b8a727 100644 --- a/apps/memos-local-plugin/core/skill/subscriber.ts +++ b/apps/memos-local-plugin/core/skill/subscriber.ts @@ -37,6 +37,7 @@ import type { SkillId } from "../types.js"; import { now as nowMs } from "../time.js"; import { IDLE_ARCHIVE_BATCH_LIMIT } from "../storage/repos/skills.js"; +/** Bound one lifecycle pass to 5,000 archival writes. */ const IDLE_ARCHIVE_MAX_BATCHES_PER_TICK = 10; export interface SkillSubscriberDeps @@ -244,7 +245,12 @@ export function attachSkillSubscriber( let archivedThisBatch = 0; for (const s of archiveCandidates) { if (!shouldArchiveIdle(s, deps.config.idleArchiveMs, deps.config, at)) continue; - deps.repos.skills.setStatus(s.id, "archived", at); + const archived = deps.repos.skills.archiveIfIdle(s.id, { + minEtaForRetrieval: deps.config.minEtaForRetrieval, + cutoff, + updatedAt: at, + }); + if (!archived) continue; archivedThisBatch += 1; archivedTotal += 1; log.info("skill.idle_archived", { @@ -269,7 +275,8 @@ export function attachSkillSubscriber( cutoff, minEtaForRetrieval: deps.config.minEtaForRetrieval, }); - break; + if (archiveCandidates.length < IDLE_ARCHIVE_BATCH_LIMIT) break; + continue; } if (archiveCandidates.length < IDLE_ARCHIVE_BATCH_LIMIT) break; if (batchesProcessed === IDLE_ARCHIVE_MAX_BATCHES_PER_TICK) { diff --git a/apps/memos-local-plugin/core/storage/repos/skills.ts b/apps/memos-local-plugin/core/storage/repos/skills.ts index 01b91eba2..c4c1fa0cc 100644 --- a/apps/memos-local-plugin/core/storage/repos/skills.ts +++ b/apps/memos-local-plugin/core/storage/repos/skills.ts @@ -63,6 +63,19 @@ export function makeSkillsRepo(db: StorageDb) { const updateStatus = db.prepare( buildUpdate({ table: "skills", columns: ["id", "status", "updated_at"] }), ); + const archiveIdle = db.prepare<{ + id: string; + min_eta: number; + cutoff: number; + updated_at: number; + }>( + `UPDATE skills + SET status = 'archived', updated_at = @updated_at + WHERE id = @id + AND status = 'active' + AND eta < @min_eta + AND COALESCE(last_used_at, created_at) <= @cutoff`, + ); const updateTrials = db.prepare( buildUpdate({ table: "skills", @@ -89,6 +102,19 @@ export function makeSkillsRepo(db: StorageDb) { updateStatus.run({ id, status, updated_at: updatedAt }); }, + archiveIfIdle( + id: SkillId, + input: { minEtaForRetrieval: number; cutoff: number; updatedAt: number }, + ): boolean { + const res = archiveIdle.run({ + id, + min_eta: input.minEtaForRetrieval, + cutoff: input.cutoff, + updated_at: input.updatedAt, + }); + return res.changes > 0; + }, + bumpTrial( id: SkillId, passed: boolean, diff --git a/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts b/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts index a8c35468a..24553238b 100644 --- a/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts @@ -219,6 +219,64 @@ describe("skill/subscriber", () => { sub.dispose(); }); + it("continues after a full batch is archived concurrently", async () => { + handle = makeTmpDb(); + const h = handle; + for (let i = 0; i < 501; i++) { + seedSkill(h, { + id: `sk_concurrent_${i}` as never, + name: `concurrent_skill_${i}`, + status: "active", + eta: 0.05, + createdAt: 1 as never, + updatedAt: (i + 1) as never, + lastUsedAt: (i + 1) as never, + }); + } + + const bus = createSkillEventBus(); + const events: string[] = []; + bus.on("skill.status.changed", (event) => { + if (event.kind === "skill.status.changed") events.push(event.skillId); + }); + const log = rootLogger.child({ channel: "core.skill.subscriber" }); + const warnSpy = vi.spyOn(log, "warn").mockImplementation(() => undefined); + const archiveIfIdle = h.repos.skills.archiveIfIdle.bind(h.repos.skills); + const archiveSpy = vi.spyOn(h.repos.skills, "archiveIfIdle").mockImplementation( + (id, input) => { + const index = Number(String(id).slice(String(id).lastIndexOf("_") + 1)); + if (index < 500) { + h.repos.skills.setStatus(id, "archived", input.updatedAt); + return false; + } + return archiveIfIdle(id, input); + }, + ); + const sub = attachSkillSubscriber({ + l2Bus: createL2EventBus(), + rewardBus: createRewardEventBus(), + bus, + repos: h.repos, + embedder: null, + llm: null, + log, + config: makeSkillConfig({ minEtaForRetrieval: 0.1, idleArchiveMs: 1_000 }), + }); + + await sub.lifecycleTick(); + + expect(h.repos.skills.count({ status: "archived" })).toBe(501); + expect(events).toEqual(["sk_concurrent_500"]); + expect(warnSpy).toHaveBeenCalledWith("skill.idle_archive_stalled", { + candidateCount: 500, + cutoff: expect.any(Number), + minEtaForRetrieval: 0.1, + }); + sub.dispose(); + archiveSpy.mockRestore(); + warnSpy.mockRestore(); + }); + it("drains more than one 500-skill idle archive batch in one lifecycle tick", async () => { handle = makeTmpDb(); const h = handle; diff --git a/apps/memos-local-plugin/tests/unit/storage/repos.test.ts b/apps/memos-local-plugin/tests/unit/storage/repos.test.ts index aef629696..24a85ad65 100644 --- a/apps/memos-local-plugin/tests/unit/storage/repos.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/repos.test.ts @@ -365,11 +365,23 @@ describe("storage/repos — happy paths", () => { expect(repos.skills.recordUse("old_used", 9_500)).toBe(true); expect(repos.skills.getById("old_used")?.lastUsedAt).toBe(9_500); + expect(repos.skills.archiveIfIdle("old_used", { + minEtaForRetrieval: 0.1, + cutoff: 9_000, + updatedAt: 10_000, + })).toBe(false); + expect(repos.skills.getById("old_used")?.status).toBe("active"); expect(repos.skills.listIdleArchiveCandidates({ minEtaForRetrieval: 0.1, cutoff: 9_000, limit: 500, }).map((skill) => skill.id)).toEqual(["never_used"]); + expect(repos.skills.archiveIfIdle("never_used", { + minEtaForRetrieval: 0.1, + cutoff: 9_000, + updatedAt: 10_000, + })).toBe(true); + expect(repos.skills.getById("never_used")?.status).toBe("archived"); } finally { cleanup(); } From 10cda282bb0dbffec4db8111905e8061d3977fe3 Mon Sep 17 00:00:00 2001 From: jiachengzhen Date: Fri, 7 Aug 2026 00:33:51 +0800 Subject: [PATCH 11/34] perf(plugin): batch idle skill archival --- .../core/skill/subscriber.ts | 27 ++++++----- .../core/storage/repos/skills.ts | 36 ++++++++------ .../tests/unit/skill/subscriber.test.ts | 23 +++++---- .../tests/unit/storage/repos.test.ts | 48 ++++++++++++++----- 4 files changed, 87 insertions(+), 47 deletions(-) diff --git a/apps/memos-local-plugin/core/skill/subscriber.ts b/apps/memos-local-plugin/core/skill/subscriber.ts index 2d2b8a727..67203b87c 100644 --- a/apps/memos-local-plugin/core/skill/subscriber.ts +++ b/apps/memos-local-plugin/core/skill/subscriber.ts @@ -25,7 +25,7 @@ import { runSkill, type RunSkillDeps, } from "./skill.js"; -import { shouldArchiveIdle, shouldPromoteCandidate } from "./lifecycle.js"; +import { shouldPromoteCandidate } from "./lifecycle.js"; import type { RunSkillInput, RunSkillResult, @@ -242,17 +242,20 @@ export function attachSkillSubscriber( limit: IDLE_ARCHIVE_BATCH_LIMIT, }); batchesProcessed += 1; - let archivedThisBatch = 0; + const archivedIds = new Set( + deps.repos.skills.archiveIdleBatch( + archiveCandidates.map((skill) => skill.id), + { + minEtaForRetrieval: deps.config.minEtaForRetrieval, + cutoff, + updatedAt: at, + }, + ), + ); + const archivedThisBatch = archivedIds.size; + archivedTotal += archivedThisBatch; for (const s of archiveCandidates) { - if (!shouldArchiveIdle(s, deps.config.idleArchiveMs, deps.config, at)) continue; - const archived = deps.repos.skills.archiveIfIdle(s.id, { - minEtaForRetrieval: deps.config.minEtaForRetrieval, - cutoff, - updatedAt: at, - }); - if (!archived) continue; - archivedThisBatch += 1; - archivedTotal += 1; + if (!archivedIds.has(s.id)) continue; log.info("skill.idle_archived", { skillId: s.id, name: s.name, @@ -270,6 +273,8 @@ export function attachSkillSubscriber( }); } if (archiveCandidates.length > 0 && archivedThisBatch === 0) { + // A full zero-change batch was invalidated by concurrent writers. + // Re-query so later eligible rows are not abandoned for this tick. log.warn("skill.idle_archive_stalled", { candidateCount: archiveCandidates.length, cutoff, diff --git a/apps/memos-local-plugin/core/storage/repos/skills.ts b/apps/memos-local-plugin/core/storage/repos/skills.ts index c4c1fa0cc..7eb493dda 100644 --- a/apps/memos-local-plugin/core/storage/repos/skills.ts +++ b/apps/memos-local-plugin/core/storage/repos/skills.ts @@ -16,6 +16,9 @@ import { export const IDLE_ARCHIVE_BATCH_LIMIT = 500; +const IDLE_ARCHIVE_PREDICATE = + "status = 'active' AND eta < @min_eta AND COALESCE(last_used_at, created_at) <= @cutoff"; + const COLUMNS = [ "id", "owner_agent_kind", @@ -72,9 +75,7 @@ export function makeSkillsRepo(db: StorageDb) { `UPDATE skills SET status = 'archived', updated_at = @updated_at WHERE id = @id - AND status = 'active' - AND eta < @min_eta - AND COALESCE(last_used_at, created_at) <= @cutoff`, + AND ${IDLE_ARCHIVE_PREDICATE}`, ); const updateTrials = db.prepare( buildUpdate({ @@ -102,17 +103,24 @@ export function makeSkillsRepo(db: StorageDb) { updateStatus.run({ id, status, updated_at: updatedAt }); }, - archiveIfIdle( - id: SkillId, + archiveIdleBatch( + ids: readonly SkillId[], input: { minEtaForRetrieval: number; cutoff: number; updatedAt: number }, - ): boolean { - const res = archiveIdle.run({ - id, - min_eta: input.minEtaForRetrieval, - cutoff: input.cutoff, - updated_at: input.updatedAt, + ): SkillId[] { + if (ids.length === 0) return []; + return db.tx(() => { + const archived: SkillId[] = []; + for (const id of ids) { + const res = archiveIdle.run({ + id, + min_eta: input.minEtaForRetrieval, + cutoff: input.cutoff, + updated_at: input.updatedAt, + }); + if (res.changes > 0) archived.push(id); + } + return archived; }); - return res.changes > 0; }, bumpTrial( @@ -192,9 +200,7 @@ export function makeSkillsRepo(db: StorageDb) { const sql = ` SELECT ${COLUMNS.join(", ")} FROM skills - WHERE status = 'active' - AND eta < @min_eta - AND COALESCE(last_used_at, created_at) <= @cutoff + WHERE ${IDLE_ARCHIVE_PREDICATE} ORDER BY COALESCE(last_used_at, created_at) ASC LIMIT @limit`; return db.prepare(sql).all(params).map(mapRow); diff --git a/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts b/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts index 24553238b..666d7f4b6 100644 --- a/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts @@ -241,17 +241,20 @@ describe("skill/subscriber", () => { }); const log = rootLogger.child({ channel: "core.skill.subscriber" }); const warnSpy = vi.spyOn(log, "warn").mockImplementation(() => undefined); - const archiveIfIdle = h.repos.skills.archiveIfIdle.bind(h.repos.skills); - const archiveSpy = vi.spyOn(h.repos.skills, "archiveIfIdle").mockImplementation( - (id, input) => { - const index = Number(String(id).slice(String(id).lastIndexOf("_") + 1)); - if (index < 500) { - h.repos.skills.setStatus(id, "archived", input.updatedAt); - return false; - } - return archiveIfIdle(id, input); - }, + const archiveIdleBatch = h.repos.skills.archiveIdleBatch.bind( + h.repos.skills, ); + const archiveSpy = vi + .spyOn(h.repos.skills, "archiveIdleBatch") + .mockImplementation((ids, input) => { + for (const id of ids) { + const index = Number(String(id).slice(String(id).lastIndexOf("_") + 1)); + if (index < 500) { + h.repos.skills.setStatus(id, "archived", input.updatedAt); + } + } + return archiveIdleBatch(ids, input); + }); const sub = attachSkillSubscriber({ l2Bus: createL2EventBus(), rewardBus: createRewardEventBus(), diff --git a/apps/memos-local-plugin/tests/unit/storage/repos.test.ts b/apps/memos-local-plugin/tests/unit/storage/repos.test.ts index 24a85ad65..ed1aa53db 100644 --- a/apps/memos-local-plugin/tests/unit/storage/repos.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/repos.test.ts @@ -315,7 +315,7 @@ describe("storage/repos — happy paths", () => { }); it("skills: selects idle archive candidates and excludes a skill after recorded use", () => { - const { repos, cleanup } = makeTmpDb(); + const { db, repos, cleanup } = makeTmpDb(); try { const insertSkill = ( id: string, @@ -365,23 +365,49 @@ describe("storage/repos — happy paths", () => { expect(repos.skills.recordUse("old_used", 9_500)).toBe(true); expect(repos.skills.getById("old_used")?.lastUsedAt).toBe(9_500); - expect(repos.skills.archiveIfIdle("old_used", { - minEtaForRetrieval: 0.1, - cutoff: 9_000, - updatedAt: 10_000, - })).toBe(false); + expect( + repos.skills.archiveIdleBatch(["old_used"], { + minEtaForRetrieval: 0.1, + cutoff: 9_000, + updatedAt: 10_000, + }), + ).toEqual([]); expect(repos.skills.getById("old_used")?.status).toBe("active"); expect(repos.skills.listIdleArchiveCandidates({ minEtaForRetrieval: 0.1, cutoff: 9_000, limit: 500, }).map((skill) => skill.id)).toEqual(["never_used"]); - expect(repos.skills.archiveIfIdle("never_used", { - minEtaForRetrieval: 0.1, - cutoff: 9_000, - updatedAt: 10_000, - })).toBe(true); + expect( + repos.skills.archiveIdleBatch(["never_used"], { + minEtaForRetrieval: 0.1, + cutoff: 9_000, + updatedAt: 10_000, + }), + ).toEqual(["never_used"]); expect(repos.skills.getById("never_used")?.status).toBe("archived"); + + insertSkill("rollback_first", "active", 0.05, 1, 100); + insertSkill("rollback_fail", "active", 0.05, 1, 100); + db.exec(` + CREATE TRIGGER reject_idle_archive + BEFORE UPDATE OF status ON skills + WHEN OLD.id = 'rollback_fail' AND NEW.status = 'archived' + BEGIN + SELECT RAISE(ABORT, 'forced archive failure'); + END`); + expect(() => + repos.skills.archiveIdleBatch( + ["rollback_first", "rollback_fail"], + { + minEtaForRetrieval: 0.1, + cutoff: 9_000, + updatedAt: 10_000, + }, + ), + ).toThrow(/forced archive failure/); + expect(repos.skills.getById("rollback_first")?.status).toBe("active"); + expect(repos.skills.getById("rollback_fail")?.status).toBe("active"); } finally { cleanup(); } From d38dfacb447b24229a5e56e41efe1e3f4d6d00ca Mon Sep 17 00:00:00 2001 From: autodev Date: Fri, 14 Aug 2026 06:45:18 +0800 Subject: [PATCH 12/34] fix(config): resolve masked apiKey from env on load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bridge persists config.yaml with API keys masked to __memos_secret__ via maskSecrets() and strips empty secrets from patches via stripEmptySecrets(), but nothing re-reads the real value back. On daemon restart, loadConfig() treats the mask as the literal API key, every LLM call fails auth, and the bridge restart-loops with lastOkAt: null and skill.crystallize stuck. Make resolveConfig() (the single choke point for both disk-loaded and in-memory patched configs) walk SECRET_FIELD_PATHS after pruneUnknown and before deepMerge: - ${VAR} references resolve from process.env when the name matches the allowlist ^[A-Z][A-Z0-9_]*_(API_KEY|TOKEN)$; other names emit a warning and stay untouched. - __memos_secret__ / empty apiKey leaves fall back to LLM_API_KEY (or EMBEDDING_API_KEY for embedding.apiKey), then to OPENCODE_GO_API_KEY / OPENCODE_ZEN_API_KEY for LLM-class fields only. Embedding never inherits an LLM provider's key. - Hub tokens (hub.teamToken, hub.userToken) require an explicit ${VAR} — no path-based env convention. - Real values pass through unchanged; the caller's raw config object is never mutated (resolution runs on the pruneUnknown copy). Read-side only: on-disk write stays masked, so the security posture of maskSecrets() is preserved. Adds 10 unit tests under tests/unit/config/resolve-secret-env.test.ts covering ${VAR} expansion, mask sentinel resolution, empty-string fallback, per-path env conventions (embedding vs LLM channel isolation), hub token ${VAR} path, allowlist enforcement, non-mutation of the raw config, and negative cases (no env → mask retained; unset ${VAR} → literal preserved; real values untouched). Fixes #2245 --- apps/memos-local-plugin/core/config/index.ts | 83 +++++++++++++- .../unit/config/resolve-secret-env.test.ts | 103 ++++++++++++++++++ 2 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 apps/memos-local-plugin/tests/unit/config/resolve-secret-env.test.ts diff --git a/apps/memos-local-plugin/core/config/index.ts b/apps/memos-local-plugin/core/config/index.ts index 6d529d960..1c7b8c5d3 100644 --- a/apps/memos-local-plugin/core/config/index.ts +++ b/apps/memos-local-plugin/core/config/index.ts @@ -18,7 +18,7 @@ import { MemosError } from "../../agent-contract/errors.js"; import type { ResolvedHome } from "./paths.js"; import { resolveHome } from "./paths.js"; import { ConfigSchema, type ResolvedConfig } from "./schema.js"; -import { DEFAULT_CONFIG, effectiveViewerPort } from "./defaults.js"; +import { DEFAULT_CONFIG, SECRET_FIELD_PATHS, effectiveViewerPort } from "./defaults.js"; import { migrateHermesViewerPort } from "./migrations.js"; import { parseYaml } from "./yaml.js"; @@ -72,9 +72,40 @@ export async function loadConfig(home: ResolvedHome, agent?: string): Promise use process.env[NAME]. Only allowlisted + // names are expanded (`^[A-Z][A-Z0-9_]*_(API_KEY|TOKEN)$`); + // anything else emits a warning and is left untouched. + // 2. Value is the mask sentinel `__memos_secret__` or empty string + // -> use the env var inferred from the field path + // (llm.apiKey -> LLM_API_KEY, then OPENCODE_GO_API_KEY / + // OPENCODE_ZEN_API_KEY fallbacks for the opencode-go/zen + // providers). The generic fallbacks only apply to LLM-class + // fields — embedding.apiKey is never handed an LLM provider's key. + // 3. Otherwise leave the value untouched. + // + // The mask itself is never used as a credential, and the on-disk write + // stays masked (security preserved); this is read-side only. + resolveSecretEnv(cleaned, warnings); const merged = deepMerge(DEFAULT_CONFIG as Record, cleaned); stripUnsupportedEmbeddingDimensions(merged); const viewerPort = effectiveViewerPort(agent); @@ -104,6 +135,56 @@ export function resolveConfig(raw: unknown, warnings?: string[], agent?: string) // ─── helpers ──────────────────────────────────────────────────────────────── +/** Env var names accepted in `${NAME}` config references. */ +const ENV_REF_ALLOWLIST = /^[A-Z][A-Z0-9_]*_(API_KEY|TOKEN)$/; + +/** + * Replace masked / empty / `${VAR}` secret leaves in `cleaned` (a freshly + * built, non-shared object — see `pruneUnknown`) with values from the + * environment. The caller's raw config object is never written to. + */ +function resolveSecretEnv(cleaned: Record, warnings?: string[]): void { + for (const dotted of SECRET_FIELD_PATHS) { + const keys = dotted.split("."); + let cursor: unknown = cleaned; + for (let i = 0; i < keys.length - 1; i++) { + if (!isPlainObject(cursor)) break; + cursor = (cursor as Record)[keys[i]!]; + } + if (!isPlainObject(cursor)) continue; + const leaf = keys[keys.length - 1]!; + const val = (cursor as Record)[leaf]; + if (typeof val !== "string") continue; + + let envName: string | null = null; + let genericFallbacks = false; + if (val.startsWith("${") && val.endsWith("}")) { + const name = val.slice(2, -1); + if (!ENV_REF_ALLOWLIST.test(name)) { + warnings?.push( + `config: leaving '${dotted}' as '${val}' — env name '${name}' is not allowlisted ` + + `(expected ^[A-Z][A-Z0-9_]*_(API_KEY|TOKEN)$)` + ); + continue; + } + envName = name; + } else if (val === "__memos_secret__" || val === "") { + if (leaf !== "apiKey") continue; + const isEmbedding = keys[keys.length - 2] === "embedding"; + envName = isEmbedding ? "EMBEDDING_API_KEY" : "LLM_API_KEY"; + genericFallbacks = !isEmbedding; + } + if (!envName) continue; + + const envVal = + process.env[envName] ?? + (genericFallbacks + ? (process.env.OPENCODE_GO_API_KEY ?? process.env.OPENCODE_ZEN_API_KEY) + : undefined); + if (envVal) (cursor as Record)[leaf] = envVal; + } +} + function formatErr(e: ValueError): string { return `${e.path || ""}: ${e.message}`; } diff --git a/apps/memos-local-plugin/tests/unit/config/resolve-secret-env.test.ts b/apps/memos-local-plugin/tests/unit/config/resolve-secret-env.test.ts new file mode 100644 index 000000000..0a7652289 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/config/resolve-secret-env.test.ts @@ -0,0 +1,103 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { resolveConfig } from "../../../core/config/index.js"; +import { SECRET_FIELD_PATHS } from "../../../core/config/defaults.js"; + +const ORIGINAL_ENV = { ...process.env }; + +afterEach(() => { + process.env = { ...ORIGINAL_ENV }; +}); + +describe("resolveConfig secret env fallback", () => { + it("expands allowlisted ${ENV_VAR} references in secret fields", () => { + process.env.MY_LLM_API_KEY = "sk-env-expanded"; + const cfg = resolveConfig({ llm: { apiKey: "${MY_LLM_API_KEY}" } }); + expect(cfg.llm.apiKey).toBe("sk-env-expanded"); + }); + + it("resolves the __memos_secret__ mask sentinel from env", () => { + process.env.OPENCODE_GO_API_KEY = "sk-mask-resolved"; + const cfg = resolveConfig({ llm: { apiKey: "__memos_secret__" } }); + expect(cfg.llm.apiKey).toBe("sk-mask-resolved"); + }); + + it("resolves empty string secret fields from env", () => { + process.env.OPENCODE_ZEN_API_KEY = "sk-empty-resolved"; + const cfg = resolveConfig({ llm: { apiKey: "" } }); + expect(cfg.llm.apiKey).toBe("sk-empty-resolved"); + }); + + it("uses per-path env conventions — embedding gets EMBEDDING_API_KEY, never an LLM key", () => { + process.env.OPENCODE_GO_API_KEY = "sk-llm"; + process.env.EMBEDDING_API_KEY = "sk-embed"; + const raw: Record = {}; + for (const dotted of SECRET_FIELD_PATHS) { + const keys = dotted.split("."); + let cursor = raw; + for (let i = 0; i < keys.length - 1; i++) { + cursor[keys[i]!] = cursor[keys[i]!] ?? {}; + cursor = cursor[keys[i]!] as Record; + } + cursor[keys[keys.length - 1]!] = "__memos_secret__"; + } + const cfg = resolveConfig(raw); + for (const dotted of SECRET_FIELD_PATHS) { + const keys = dotted.split("."); + let cursor: unknown = cfg; + for (const k of keys) { + cursor = (cursor as Record)[k]; + } + if (dotted === "embedding.apiKey") { + expect(cursor).toBe("sk-embed"); + } else if (dotted.endsWith("apiKey")) { + expect(cursor).toBe("sk-llm"); + } else { + expect(cursor).toBe("__memos_secret__"); + } + } + }); + + it("resolves hub tokens via explicit ${VAR} references", () => { + process.env.HUB_TEAM_TOKEN = "sk-hub-token"; + const cfg = resolveConfig({ hub: { teamToken: "${HUB_TEAM_TOKEN}" } }); + expect(cfg.hub.teamToken).toBe("sk-hub-token"); + }); + + it("does not fall back to generic keys when an explicit ${VAR} is unset", () => { + process.env.OPENCODE_GO_API_KEY = "sk-llm"; + process.env.OPENCODE_ZEN_API_KEY = "sk-zen"; + const cfg = resolveConfig({ llm: { apiKey: "${MY_LLM_API_KEY}" } }); + expect(cfg.llm.apiKey).toBe("${MY_LLM_API_KEY}"); + }); + + it("warns and skips expansion for non-allowlisted ${VAR} names", () => { + process.env.HOME = "/home/test"; + const warnings: string[] = []; + const cfg = resolveConfig({ llm: { apiKey: "${HOME}" } }, warnings); + expect(cfg.llm.apiKey).toBe("${HOME}"); + expect(warnings.length).toBe(1); + expect(warnings[0]).toContain("not allowlisted"); + }); + + it("leaves real (non-placeholder) values untouched", () => { + const cfg = resolveConfig({ llm: { apiKey: "sk-real-value" } }); + expect(cfg.llm.apiKey).toBe("sk-real-value"); + }); + + it("leaves placeholders untouched when no env var is set", () => { + delete process.env.LLM_API_KEY; + delete process.env.OPENCODE_GO_API_KEY; + delete process.env.OPENCODE_ZEN_API_KEY; + const cfg = resolveConfig({ llm: { apiKey: "__memos_secret__" } }); + expect(cfg.llm.apiKey).toBe("__memos_secret__"); + }); + + it("never mutates the caller's raw config object", () => { + process.env.OPENCODE_GO_API_KEY = "sk-llm"; + const raw = { llm: { apiKey: "__memos_secret__" } }; + const cfg = resolveConfig(raw); + expect(cfg.llm.apiKey).toBe("sk-llm"); + expect(raw.llm.apiKey).toBe("__memos_secret__"); + }); +}); From 1f28c8c57ee29a880ad2ca0bfa9754f860e89cbd Mon Sep 17 00:00:00 2001 From: autodev Date: Fri, 14 Aug 2026 07:00:05 +0800 Subject: [PATCH 13/34] fix(config): apply OCR review to secret env resolver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address 4 findings from the open-code-review pass on PR #2246: 1. hub.teamToken / hub.userToken are now resolved from the environment when masked with __memos_secret__ or written as empty strings. The previous `if (leaf !== "apiKey") continue` short-circuit silently perpetuated the original bug for hub tokens. 2. Emit a warning when a secret leaf references an env var that is not set (both the explicit ${VAR} form and the mask/empty form). Without this, a user who writes `apiKey: ${MY_API_KEY}` and forgets to export MY_API_KEY sees auth failures with no actionable log line. 3. Restrict the OPENCODE_GO_API_KEY / OPENCODE_ZEN_API_KEY generic fallback to the primary llm.apiKey. Per-component overrides (l3Llm.apiKey, skillEvolver.apiKey) and non-LLM secrets (embedding.apiKey, hub.*Token) must never silently borrow an unrelated provider's key — that causes cross-provider auth failures and unexpected billing when those components are pointed at a different provider than the shared llm settings. 4. Use an explicit `traversalOk` flag when walking SECRET_FIELD_PATHS so a partial traversal cannot leave `cursor` pointing at a shallower valid intermediate node that would then pass the isPlainObject check and cause `leaf` to be looked up on the wrong object. Today every entry is 2 levels deep so the bug is latent, but the flag makes the intent explicit and future-proofs against deeper paths being added. Env var derivation for masked/empty leaves now uses a camel→SNAKE transform on the last two path segments so every SECRET_FIELD_PATHS entry is resolvable by convention: embedding.apiKey → EMBEDDING_API_KEY llm.apiKey → LLM_API_KEY l3Llm.apiKey → L3_LLM_API_KEY skillEvolver.apiKey → SKILL_EVOLVER_API_KEY hub.teamToken → HUB_TEAM_TOKEN hub.userToken → HUB_USER_TOKEN Tests updated to cover the new hub-token resolution, the tightened fallback scope, and both warning cases. All 76 config tests pass; tsc --noEmit clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/memos-local-plugin/core/config/index.ts | 81 ++++++++++++++++--- .../unit/config/resolve-secret-env.test.ts | 81 +++++++++++++++++-- 2 files changed, 142 insertions(+), 20 deletions(-) diff --git a/apps/memos-local-plugin/core/config/index.ts b/apps/memos-local-plugin/core/config/index.ts index 1c7b8c5d3..e188c2abb 100644 --- a/apps/memos-local-plugin/core/config/index.ts +++ b/apps/memos-local-plugin/core/config/index.ts @@ -96,13 +96,20 @@ export function resolveConfig(raw: unknown, warnings?: string[], agent?: string) // names are expanded (`^[A-Z][A-Z0-9_]*_(API_KEY|TOKEN)$`); // anything else emits a warning and is left untouched. // 2. Value is the mask sentinel `__memos_secret__` or empty string - // -> use the env var inferred from the field path - // (llm.apiKey -> LLM_API_KEY, then OPENCODE_GO_API_KEY / - // OPENCODE_ZEN_API_KEY fallbacks for the opencode-go/zen - // providers). The generic fallbacks only apply to LLM-class - // fields — embedding.apiKey is never handed an LLM provider's key. + // -> derive the env var from the field path itself + // (llm.apiKey -> LLM_API_KEY, hub.teamToken -> HUB_TEAM_TOKEN, + // skillEvolver.apiKey -> SKILL_EVOLVER_API_KEY, …). The generic + // OPENCODE_GO/ZEN fallback is applied ONLY to the primary + // `llm.apiKey`; per-component overrides (l3Llm, skillEvolver) + // and non-LLM secrets (embedding, hub tokens) never borrow an + // unrelated provider's key — that would cause cross-provider + // auth failures or unexpected billing on the wrong account. // 3. Otherwise leave the value untouched. // + // Any secret leaf that references an env var that is not set emits a + // warning so the operator gets an actionable log message instead of + // silent auth failures on the next LLM call. + // // The mask itself is never used as a credential, and the on-disk write // stays masked (security preserved); this is read-side only. resolveSecretEnv(cleaned, warnings); @@ -147,11 +154,21 @@ function resolveSecretEnv(cleaned: Record, warnings?: string[]) for (const dotted of SECRET_FIELD_PATHS) { const keys = dotted.split("."); let cursor: unknown = cleaned; + // Explicit flag: `break` alone leaves `cursor` pointing at the last + // valid value, which for paths deeper than 2 levels could accidentally + // pass the `isPlainObject(cursor)` check below and index `leaf` on the + // wrong node. Today every SECRET_FIELD_PATHS entry is only 2 levels + // deep, but keeping the flag makes the intent explicit and future- + // proofs against deeper paths being added. + let traversalOk = true; for (let i = 0; i < keys.length - 1; i++) { - if (!isPlainObject(cursor)) break; + if (!isPlainObject(cursor)) { + traversalOk = false; + break; + } cursor = (cursor as Record)[keys[i]!]; } - if (!isPlainObject(cursor)) continue; + if (!traversalOk || !isPlainObject(cursor)) continue; const leaf = keys[keys.length - 1]!; const val = (cursor as Record)[leaf]; if (typeof val !== "string") continue; @@ -169,10 +186,25 @@ function resolveSecretEnv(cleaned: Record, warnings?: string[]) } envName = name; } else if (val === "__memos_secret__" || val === "") { - if (leaf !== "apiKey") continue; - const isEmbedding = keys[keys.length - 2] === "embedding"; - envName = isEmbedding ? "EMBEDDING_API_KEY" : "LLM_API_KEY"; - genericFallbacks = !isEmbedding; + // Derive env var name from the field path itself so every entry + // in SECRET_FIELD_PATHS is resolvable, not just the ones whose + // leaf is `apiKey`: + // embedding.apiKey → EMBEDDING_API_KEY + // llm.apiKey → LLM_API_KEY + // l3Llm.apiKey → L3_LLM_API_KEY + // skillEvolver.apiKey → SKILL_EVOLVER_API_KEY + // hub.teamToken → HUB_TEAM_TOKEN + // hub.userToken → HUB_USER_TOKEN + const parent = keys[keys.length - 2] ?? ""; + envName = `${camelToUpperSnake(parent)}_${camelToUpperSnake(leaf)}`; + // OPENCODE_GO_API_KEY / OPENCODE_ZEN_API_KEY are only meaningful + // for the primary `llm.apiKey`. Per-component overrides + // (l3Llm.apiKey, skillEvolver.apiKey) and non-LLM secrets + // (embedding.apiKey, hub.*Token) must not silently borrow an + // unrelated provider's key — doing so causes cross-provider auth + // failures and unexpected billing on the wrong account when the + // component is configured for a different provider entirely. + genericFallbacks = parent === "llm" && leaf === "apiKey"; } if (!envName) continue; @@ -181,10 +213,35 @@ function resolveSecretEnv(cleaned: Record, warnings?: string[]) (genericFallbacks ? (process.env.OPENCODE_GO_API_KEY ?? process.env.OPENCODE_ZEN_API_KEY) : undefined); - if (envVal) (cursor as Record)[leaf] = envVal; + if (envVal) { + (cursor as Record)[leaf] = envVal; + } else { + // Explicit-reference case: the user asked for env expansion but + // the target is unset. Mask/empty case: we walked the path-based + // convention and nothing was set. Both perpetuate the original + // bug (silent auth failure on next LLM call) unless we log it. + warnings?.push( + `config: '${dotted}' references env var '${envName}' but it is not set — ` + + `field left as placeholder and auth will fail on the next call` + ); + } } } +/** + * camelCase → UPPER_SNAKE_CASE for deriving env var names from config + * field paths. Only inserts an underscore at a lowercase/digit → uppercase + * boundary so acronyms and digit runs stay intact: + * apiKey → API_KEY + * teamToken → TEAM_TOKEN + * userToken → USER_TOKEN + * l3Llm → L3_LLM + * skillEvolver → SKILL_EVOLVER + */ +function camelToUpperSnake(s: string): string { + return s.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toUpperCase(); +} + function formatErr(e: ValueError): string { return `${e.path || ""}: ${e.message}`; } diff --git a/apps/memos-local-plugin/tests/unit/config/resolve-secret-env.test.ts b/apps/memos-local-plugin/tests/unit/config/resolve-secret-env.test.ts index 0a7652289..9054010a2 100644 --- a/apps/memos-local-plugin/tests/unit/config/resolve-secret-env.test.ts +++ b/apps/memos-local-plugin/tests/unit/config/resolve-secret-env.test.ts @@ -28,9 +28,16 @@ describe("resolveConfig secret env fallback", () => { expect(cfg.llm.apiKey).toBe("sk-empty-resolved"); }); - it("uses per-path env conventions — embedding gets EMBEDDING_API_KEY, never an LLM key", () => { + it("uses per-path env conventions — every secret path resolves from its own env var", () => { + // OPENCODE_GO_API_KEY is only the generic fallback for the *primary* + // llm.apiKey — l3Llm / skillEvolver / hub / embedding all get their + // own path-derived env var and never silently borrow the LLM key. process.env.OPENCODE_GO_API_KEY = "sk-llm"; process.env.EMBEDDING_API_KEY = "sk-embed"; + process.env.L3_LLM_API_KEY = "sk-l3"; + process.env.SKILL_EVOLVER_API_KEY = "sk-skill"; + process.env.HUB_TEAM_TOKEN = "sk-team"; + process.env.HUB_USER_TOKEN = "sk-user"; const raw: Record = {}; for (const dotted of SECRET_FIELD_PATHS) { const keys = dotted.split("."); @@ -42,22 +49,80 @@ describe("resolveConfig secret env fallback", () => { cursor[keys[keys.length - 1]!] = "__memos_secret__"; } const cfg = resolveConfig(raw); + const expected: Record = { + "embedding.apiKey": "sk-embed", + "llm.apiKey": "sk-llm", + "l3Llm.apiKey": "sk-l3", + "skillEvolver.apiKey": "sk-skill", + "hub.teamToken": "sk-team", + "hub.userToken": "sk-user", + }; for (const dotted of SECRET_FIELD_PATHS) { const keys = dotted.split("."); let cursor: unknown = cfg; for (const k of keys) { cursor = (cursor as Record)[k]; } - if (dotted === "embedding.apiKey") { - expect(cursor).toBe("sk-embed"); - } else if (dotted.endsWith("apiKey")) { - expect(cursor).toBe("sk-llm"); - } else { - expect(cursor).toBe("__memos_secret__"); - } + expect(cursor).toBe(expected[dotted]); } }); + it("resolves masked hub tokens from HUB_TEAM_TOKEN / HUB_USER_TOKEN", () => { + // Regression: previously the mask/empty path only ran when `leaf === + // 'apiKey'`, so hub.teamToken / hub.userToken masked by + // maskSecrets() were silently left unresolved and hub auth failed + // exactly like the LLM auth bug in #2245. + process.env.HUB_TEAM_TOKEN = "sk-team-mask"; + process.env.HUB_USER_TOKEN = "sk-user-empty"; + const cfg = resolveConfig({ + hub: { teamToken: "__memos_secret__", userToken: "" }, + }); + expect(cfg.hub.teamToken).toBe("sk-team-mask"); + expect(cfg.hub.userToken).toBe("sk-user-empty"); + }); + + it("does not fall back to OPENCODE_GO/ZEN for l3Llm.apiKey", () => { + // Per-component overrides must not silently inherit the primary + // provider's key: l3-llm and skill-evolver are frequently pointed + // at a different provider than the shared llm settings. + process.env.OPENCODE_GO_API_KEY = "sk-llm"; + process.env.OPENCODE_ZEN_API_KEY = "sk-zen"; + delete process.env.L3_LLM_API_KEY; + const cfg = resolveConfig({ l3Llm: { apiKey: "__memos_secret__" } }); + expect(cfg.l3Llm.apiKey).toBe("__memos_secret__"); + }); + + it("does not fall back to OPENCODE_GO/ZEN for skillEvolver.apiKey", () => { + process.env.OPENCODE_GO_API_KEY = "sk-llm"; + process.env.OPENCODE_ZEN_API_KEY = "sk-zen"; + delete process.env.SKILL_EVOLVER_API_KEY; + const cfg = resolveConfig({ skillEvolver: { apiKey: "__memos_secret__" } }); + expect(cfg.skillEvolver.apiKey).toBe("__memos_secret__"); + }); + + it("warns when an explicit ${VAR} reference cannot be resolved", () => { + // Without a warning the user sees auth failures with no actionable + // hint; the whole point of the read-side resolver is to make config + // → env misconfiguration debuggable. + delete process.env.MISSING_LLM_API_KEY; + const warnings: string[] = []; + const cfg = resolveConfig({ llm: { apiKey: "${MISSING_LLM_API_KEY}" } }, warnings); + expect(cfg.llm.apiKey).toBe("${MISSING_LLM_API_KEY}"); + expect(warnings.some((w) => w.includes("MISSING_LLM_API_KEY") && w.includes("not set"))).toBe( + true, + ); + }); + + it("warns when a masked apiKey has no backing env var", () => { + delete process.env.LLM_API_KEY; + delete process.env.OPENCODE_GO_API_KEY; + delete process.env.OPENCODE_ZEN_API_KEY; + const warnings: string[] = []; + const cfg = resolveConfig({ llm: { apiKey: "__memos_secret__" } }, warnings); + expect(cfg.llm.apiKey).toBe("__memos_secret__"); + expect(warnings.some((w) => w.includes("llm.apiKey") && w.includes("not set"))).toBe(true); + }); + it("resolves hub tokens via explicit ${VAR} references", () => { process.env.HUB_TEAM_TOKEN = "sk-hub-token"; const cfg = resolveConfig({ hub: { teamToken: "${HUB_TEAM_TOKEN}" } }); From 70646de30588f713b784d8009998a3b2fa4a1c69 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Fri, 14 Aug 2026 01:21:04 +0000 Subject: [PATCH 14/34] fix(plugin): declare llm.maxTokens and llm.headers as first-class config keys llm.maxTokens and llm.headers are read at runtime (client.ts reads config.maxTokens, providers spread config.headers) but were absent from DEFAULT_CONFIG and LlmSchema/SkillEvolverSchema, so every boot logged "unknown config key 'llm.maxTokens'" and "unknown config key 'llm.headers.'" (pruneUnknown recursed into the empty headers slot and warned per user key). Add both to defaults + schema, and teach pruneUnknown that an empty-object default slot is a free-form map that must be kept as-is, eliminating the per-key warnings. Adds a regression test covering acceptance, defaults, range validation and the free-form-map warning suppression. --- .../core/config/defaults.ts | 3 + apps/memos-local-plugin/core/config/index.ts | 7 +++ apps/memos-local-plugin/core/config/schema.ts | 6 ++ .../config/llm-max-tokens-headers.test.ts | 62 +++++++++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts diff --git a/apps/memos-local-plugin/core/config/defaults.ts b/apps/memos-local-plugin/core/config/defaults.ts index 6f06210d0..a52678453 100644 --- a/apps/memos-local-plugin/core/config/defaults.ts +++ b/apps/memos-local-plugin/core/config/defaults.ts @@ -57,6 +57,8 @@ export const DEFAULT_CONFIG: ResolvedConfig = { providerIgnore: [], providerOrder: [], openRouter: false, + maxTokens: 1024, + headers: {}, }, l3Llm: { // Empty by default — falls back to the shared `llm` settings. @@ -87,6 +89,7 @@ export const DEFAULT_CONFIG: ResolvedConfig = { providerIgnore: [], providerOrder: [], openRouter: false, + maxTokens: 1024, }, storage: { ftsTokenizer: "trigram", diff --git a/apps/memos-local-plugin/core/config/index.ts b/apps/memos-local-plugin/core/config/index.ts index 6d529d960..64370f5ad 100644 --- a/apps/memos-local-plugin/core/config/index.ts +++ b/apps/memos-local-plugin/core/config/index.ts @@ -165,6 +165,13 @@ function pruneUnknown( continue; } if (isPlainObject(v) && isPlainObject((defaults as Record)[k])) { + if (Object.keys((defaults as Record)[k] as Record).length === 0) { + // Empty-object default slot = free-form map (e.g. llm.headers, a + // Record). Keep the whole user object as-is; recursing + // would warn on every user key. + out[k] = v; + continue; + } out[k] = pruneUnknown(v, (defaults as Record)[k], path, warnings); } else { out[k] = v; diff --git a/apps/memos-local-plugin/core/config/schema.ts b/apps/memos-local-plugin/core/config/schema.ts index 8566f90f3..5fdadc740 100644 --- a/apps/memos-local-plugin/core/config/schema.ts +++ b/apps/memos-local-plugin/core/config/schema.ts @@ -99,6 +99,10 @@ const LlmSchema = Type.Object({ openRouter: Type.Optional(Bool(false)), /** Optional reasoning control (see ReasoningSchema). Omit = model default. */ reasoning: Type.Optional(ReasoningSchema), + /** Max output tokens per completion (deepseek-v4-flash needs >= 100). */ + maxTokens: NumberInRange(1024, 16, 131072), + /** Extra HTTP headers for the provider request. */ + headers: Type.Optional(Type.Record(Type.String(), Type.String(), { default: {} })), }, { default: {} }); /** @@ -131,6 +135,8 @@ const SkillEvolverSchema = Type.Object({ openRouter: Type.Optional(Bool(false)), /** Optional reasoning control (see ReasoningSchema). Omit = model default. */ reasoning: Type.Optional(ReasoningSchema), + /** Max output tokens per completion. */ + maxTokens: NumberInRange(1024, 16, 131072), }, { default: {} }); const StorageSchema = Type.Object({ diff --git a/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts b/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts new file mode 100644 index 000000000..daae23de6 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; + +import { DEFAULT_CONFIG, resolveConfig } from "../../../core/config/index.js"; + +describe("resolveConfig llm.maxTokens + llm.headers", () => { + it("accepts llm.maxTokens and llm.headers without unknown-key warnings", () => { + const warnings: string[] = []; + const cfg = resolveConfig( + { + llm: { + maxTokens: 2048, + headers: { "User-Agent": "hermes-test", "X-Custom": "v1" }, + }, + }, + warnings, + ); + expect(cfg.llm.maxTokens).toBe(2048); + expect(cfg.llm.headers).toEqual({ "User-Agent": "hermes-test", "X-Custom": "v1" }); + // The free-form-map special case must not warn per header key. + expect(warnings).toEqual([]); + }); + + it("declares llm.maxTokens with a sane default of 1024", () => { + expect(DEFAULT_CONFIG.llm.maxTokens).toBe(1024); + const cfg = resolveConfig({}); + expect(cfg.llm.maxTokens).toBe(1024); + }); + + it("declares llm.headers defaulting to an empty map", () => { + expect(DEFAULT_CONFIG.llm.headers).toEqual({}); + const cfg = resolveConfig({}); + expect(cfg.llm.headers).toEqual({}); + }); + + it("declares skillEvolver.maxTokens (default 1024) for the crystallizer LLM slot", () => { + expect(DEFAULT_CONFIG.skillEvolver.maxTokens).toBe(1024); + const cfg = resolveConfig({ skillEvolver: { maxTokens: 4096 } }); + expect(cfg.skillEvolver.maxTokens).toBe(4096); + }); + + it("rejects out-of-range maxTokens with config_invalid", () => { + expect(() => resolveConfig({ llm: { maxTokens: 8 } })).toThrow(/config failed schema validation/); + }); + + it("rejects non-string header values", () => { + expect(() => resolveConfig({ llm: { headers: { "X-Bad": 42 } } })).toThrow( + /config failed schema validation/, + ); + }); + + it("keeps unrelated llm fields untouched when maxTokens/headers are set", () => { + const cfg = resolveConfig({ + llm: { provider: "openai_compatible", model: "deepseek-v4-flash", maxTokens: 2048 }, + }); + expect(cfg.llm.provider).toBe("openai_compatible"); + expect(cfg.llm.model).toBe("deepseek-v4-flash"); + expect(cfg.llm.temperature).toBe(0); + expect(cfg.llm.fallbackToHost).toBe(true); + expect(cfg.llm.timeoutMs).toBe(45_000); + expect(cfg.llm.maxRetries).toBe(3); + }); +}); From d59fe83afeee1f03f1bb68199c26a975280ed19e Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Fri, 14 Aug 2026 01:22:12 +0000 Subject: [PATCH 15/34] fix(plugin): add l3Llm.maxTokens default (shares SkillEvolverSchema) --- apps/memos-local-plugin/core/config/defaults.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/memos-local-plugin/core/config/defaults.ts b/apps/memos-local-plugin/core/config/defaults.ts index a52678453..52ba2a430 100644 --- a/apps/memos-local-plugin/core/config/defaults.ts +++ b/apps/memos-local-plugin/core/config/defaults.ts @@ -75,6 +75,7 @@ export const DEFAULT_CONFIG: ResolvedConfig = { providerIgnore: [], providerOrder: [], openRouter: false, + maxTokens: 1024, }, skillEvolver: { // Empty by default — falls back to the shared `llm` settings. From 10786401441a8300341aae191c494d7f70460552 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Sat, 15 Aug 2026 08:00:24 +1200 Subject: [PATCH 16/34] fix(plugin): wire l3Llm/skillEvolver maxTokens+headers; raise dedicated-slot defaults to 4096 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses OpenCodeReview feedback on #2248: - The l3Llm and skillEvolver client builders constructed their clients with explicit field picks that dropped maxTokens and headers — the config keys declared by the previous commit were inert at runtime (effective cap was the hard-coded DEFAULT_MAX_TOKENS=1024 in client.ts regardless of config). - Add maxTokens+headers to DedicatedLlmConfig and pass both through in the reflectLlm (skillEvolver) and l3Llm builders so configured values actually reach the provider request. - Add headers to SkillEvolverSchema (l3Llm/skillEvolver slots) so custom HTTP headers are accepted on those slots, mirroring the llm slot. - Raise l3Llm/skillEvolver maxTokens defaults from 1024 to 4096: L3 world- model bodies span multiple L2 policies/evidence traces, and crystallized skill bodies include invocation guides + procedure steps — 1024 tokens risks silent truncation on both workloads (both slots already assume 60s timeouts, implying heavier calls). - Update config tests: default assertions now pin 4096, plus new coverage for l3Llm.maxTokens and headers on both dedicated slots. --- .../core/config/defaults.ts | 4 ++-- apps/memos-local-plugin/core/config/schema.ts | 2 ++ .../core/pipeline/memory-core.ts | 8 +++++++ .../config/llm-max-tokens-headers.test.ts | 24 +++++++++++++++---- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/apps/memos-local-plugin/core/config/defaults.ts b/apps/memos-local-plugin/core/config/defaults.ts index 52ba2a430..a043b4cd4 100644 --- a/apps/memos-local-plugin/core/config/defaults.ts +++ b/apps/memos-local-plugin/core/config/defaults.ts @@ -75,7 +75,7 @@ export const DEFAULT_CONFIG: ResolvedConfig = { providerIgnore: [], providerOrder: [], openRouter: false, - maxTokens: 1024, + maxTokens: 4096, }, skillEvolver: { // Empty by default — falls back to the shared `llm` settings. @@ -90,7 +90,7 @@ export const DEFAULT_CONFIG: ResolvedConfig = { providerIgnore: [], providerOrder: [], openRouter: false, - maxTokens: 1024, + maxTokens: 4096, }, storage: { ftsTokenizer: "trigram", diff --git a/apps/memos-local-plugin/core/config/schema.ts b/apps/memos-local-plugin/core/config/schema.ts index 5fdadc740..ff9c1f319 100644 --- a/apps/memos-local-plugin/core/config/schema.ts +++ b/apps/memos-local-plugin/core/config/schema.ts @@ -137,6 +137,8 @@ const SkillEvolverSchema = Type.Object({ reasoning: Type.Optional(ReasoningSchema), /** Max output tokens per completion. */ maxTokens: NumberInRange(1024, 16, 131072), + /** Extra HTTP headers for the provider request. */ + headers: Type.Optional(Type.Record(Type.String(), Type.String(), { default: {} })), }, { default: {} }); const StorageSchema = Type.Object({ diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index 01402a86b..76e65d700 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -137,6 +137,10 @@ type DedicatedLlmConfig = { providerOrder?: string[]; openRouter?: boolean; reasoning?: ReasoningConfig; + /** Max output tokens per completion. */ + maxTokens?: number; + /** Extra HTTP headers for the provider request. */ + headers?: Record; }; export interface BootstrapOptions { @@ -437,6 +441,8 @@ export async function bootstrapMemoryCoreFull( providerOrder: evolver?.providerOrder, openRouter: evolver?.openRouter ?? false, reasoning: evolver?.reasoning, + maxTokens: evolver?.maxTokens, + headers: evolver?.headers, maxRetries: 3, // V7 §0.x — when the user's dedicated skill-evolver model is // down (auth, model name typo, server outage), prefer falling @@ -499,6 +505,8 @@ export async function bootstrapMemoryCoreFull( providerOrder: l3c?.providerOrder, openRouter: l3c?.openRouter ?? false, reasoning: l3c?.reasoning, + maxTokens: l3c?.maxTokens, + headers: l3c?.headers, maxRetries: 3, fallbackToHost: true, onError: (d: { provider: string; model: string; message: string; code?: string; at?: number }) => diff --git a/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts b/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts index daae23de6..40e929a78 100644 --- a/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts +++ b/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts @@ -32,10 +32,26 @@ describe("resolveConfig llm.maxTokens + llm.headers", () => { expect(cfg.llm.headers).toEqual({}); }); - it("declares skillEvolver.maxTokens (default 1024) for the crystallizer LLM slot", () => { - expect(DEFAULT_CONFIG.skillEvolver.maxTokens).toBe(1024); - const cfg = resolveConfig({ skillEvolver: { maxTokens: 4096 } }); - expect(cfg.skillEvolver.maxTokens).toBe(4096); + it("declares skillEvolver.maxTokens (default 4096) for the crystallizer LLM slot", () => { + expect(DEFAULT_CONFIG.skillEvolver.maxTokens).toBe(4096); + const cfg = resolveConfig({ skillEvolver: { maxTokens: 8192 } }); + expect(cfg.skillEvolver.maxTokens).toBe(8192); + }); + + it("declares l3Llm.maxTokens (default 4096) sharing the SkillEvolver schema", () => { + expect(DEFAULT_CONFIG.l3Llm.maxTokens).toBe(4096); + const cfg = resolveConfig({ l3Llm: { maxTokens: 8192 } }); + expect(cfg.l3Llm.maxTokens).toBe(8192); + }); + + it("accepts headers on skillEvolver/l3Llm slots (shared SkillEvolverSchema)", () => { + const cfg = resolveConfig({ + skillEvolver: { headers: { "X-Evolver": "v1" } }, + l3Llm: { headers: { "X-L3": "v2" } }, + }); + expect(cfg.skillEvolver.headers).toEqual({ "X-Evolver": "v1" }); + expect(cfg.l3Llm.headers).toEqual({ "X-L3": "v2" }); + expect(cfg.l3Llm.maxTokens).toBe(4096); }); it("rejects out-of-range maxTokens with config_invalid", () => { From 5fd491716a26a8d12e5376295eec265166ac32ed Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Sat, 15 Aug 2026 04:25:54 +0000 Subject: [PATCH 17/34] fix(core): bound startup recovery wait in shutdown to 15s core.shutdown() awaited startupRecoveryPromise with no timeout. With a large dirty episode and a slow/flaky LLM, the recovery reflect chain can take minutes, holding shutdown hostage until the systemd kill timer (observed 15 Aug 2026: SIGTERM 08:00:23 -> SIGKILL 08:10:23, 10-minute stop-sigterm wedge). Recovery is resumable: dirty episodes carry rewardDirty.failedAttempts and the periodic rescore re-runs them, so nothing is lost by proceeding after a short grace. The 15s bound still covers the fast init->shutdown SQLite race (issue #1808) that the wait was introduced for. --- apps/memos-local-plugin/core/pipeline/memory-core.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index 01402a86b..fbef633d4 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -1933,7 +1933,14 @@ export function createMemoryCore( // gateway reload would close SQLite while reflect / reward is // mid-flush, producing `SQLITE_MISUSE` noise on the way down. try { - await startupRecoveryPromise; + // Bound the wait: a slow / flaky LLM during startup recovery of a + // large dirty episode must not hold shutdown hostage until the + // systemd kill timer (15 Aug 2026 stop-sigterm wedge: SIGTERM at + // 08:00:23, SIGKILL at 08:10:23). Recovery is resumable — dirty + // episodes carry rewardDirty.failedAttempts and the periodic + // rescore re-runs them — so nothing is lost by proceeding after a + // short grace. Fast init→shutdown races still get their grace. + await withTimeout(startupRecoveryPromise, 15_000, "startup_recovery_shutdown_timeout"); } catch { /* already logged inside the recovery promise */ } From 2b7269857ad74f1dce3823b60045a261cd91cca0 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Sat, 15 Aug 2026 05:05:16 +0000 Subject: [PATCH 18/34] fix(plugin): default headers {} on l3Llm/skillEvolver slots; maxTokens floor 100 Addresses remaining OpenCodeReview feedback on #2248: headers was declared on SkillEvolverSchema but absent from the l3Llm/skillEvolver defaults, so setting those keys in YAML still warned unknown config key and bypassed the pruneUnknown free-form-map shortcut; maxTokens floor raised 16 to 100 to match the documented deepseek-v4-flash constraint; dedicated-slot headers now asserted warning-free, defaults pinned, out-of-range regression pinned at 50. --- .../core/config/defaults.ts | 2 ++ apps/memos-local-plugin/core/config/schema.ts | 4 ++-- .../config/llm-max-tokens-headers.test.ts | 22 ++++++++++++++----- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/apps/memos-local-plugin/core/config/defaults.ts b/apps/memos-local-plugin/core/config/defaults.ts index a043b4cd4..f30b00cfb 100644 --- a/apps/memos-local-plugin/core/config/defaults.ts +++ b/apps/memos-local-plugin/core/config/defaults.ts @@ -76,6 +76,7 @@ export const DEFAULT_CONFIG: ResolvedConfig = { providerOrder: [], openRouter: false, maxTokens: 4096, + headers: {}, }, skillEvolver: { // Empty by default — falls back to the shared `llm` settings. @@ -91,6 +92,7 @@ export const DEFAULT_CONFIG: ResolvedConfig = { providerOrder: [], openRouter: false, maxTokens: 4096, + headers: {}, }, storage: { ftsTokenizer: "trigram", diff --git a/apps/memos-local-plugin/core/config/schema.ts b/apps/memos-local-plugin/core/config/schema.ts index ff9c1f319..edbc6b40c 100644 --- a/apps/memos-local-plugin/core/config/schema.ts +++ b/apps/memos-local-plugin/core/config/schema.ts @@ -100,7 +100,7 @@ const LlmSchema = Type.Object({ /** Optional reasoning control (see ReasoningSchema). Omit = model default. */ reasoning: Type.Optional(ReasoningSchema), /** Max output tokens per completion (deepseek-v4-flash needs >= 100). */ - maxTokens: NumberInRange(1024, 16, 131072), + maxTokens: NumberInRange(1024, 100, 131072), /** Extra HTTP headers for the provider request. */ headers: Type.Optional(Type.Record(Type.String(), Type.String(), { default: {} })), }, { default: {} }); @@ -136,7 +136,7 @@ const SkillEvolverSchema = Type.Object({ /** Optional reasoning control (see ReasoningSchema). Omit = model default. */ reasoning: Type.Optional(ReasoningSchema), /** Max output tokens per completion. */ - maxTokens: NumberInRange(1024, 16, 131072), + maxTokens: NumberInRange(1024, 100, 131072), /** Extra HTTP headers for the provider request. */ headers: Type.Optional(Type.Record(Type.String(), Type.String(), { default: {} })), }, { default: {} }); diff --git a/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts b/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts index 40e929a78..19f424856 100644 --- a/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts +++ b/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts @@ -44,18 +44,28 @@ describe("resolveConfig llm.maxTokens + llm.headers", () => { expect(cfg.l3Llm.maxTokens).toBe(8192); }); - it("accepts headers on skillEvolver/l3Llm slots (shared SkillEvolverSchema)", () => { - const cfg = resolveConfig({ - skillEvolver: { headers: { "X-Evolver": "v1" } }, - l3Llm: { headers: { "X-L3": "v2" } }, - }); + it("accepts headers on skillEvolver/l3Llm slots without unknown-key warnings", () => { + const warnings: string[] = []; + const cfg = resolveConfig( + { + skillEvolver: { headers: { "X-Evolver": "v1" } }, + l3Llm: { headers: { "X-L3": "v2" } }, + }, + warnings, + ); expect(cfg.skillEvolver.headers).toEqual({ "X-Evolver": "v1" }); expect(cfg.l3Llm.headers).toEqual({ "X-L3": "v2" }); expect(cfg.l3Llm.maxTokens).toBe(4096); + expect(warnings).toEqual([]); + }); + + it("declares headers defaulting to empty on the dedicated slots", () => { + expect(DEFAULT_CONFIG.skillEvolver.headers).toEqual({}); + expect(DEFAULT_CONFIG.l3Llm.headers).toEqual({}); }); it("rejects out-of-range maxTokens with config_invalid", () => { - expect(() => resolveConfig({ llm: { maxTokens: 8 } })).toThrow(/config failed schema validation/); + expect(() => resolveConfig({ llm: { maxTokens: 50 } })).toThrow(/config failed schema validation/); }); it("rejects non-string header values", () => { From 763ca1fcd97ff325640ced9b70f5b91fcd68b928 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Sat, 15 Aug 2026 05:11:17 +0000 Subject: [PATCH 19/34] fix(skill): auto-generate crystallizer summary/steps instead of rejecting defaultDraftValidator threw skill.crystallize.invalid: missing summary / missing steps whenever the LLM returned valid JSON without those fields (observed with deepseek-v4-flash), flooding bridge logs every 5-10s and stalling the crystallizer queue. It now repairs the draft instead: - missing summary: derived from first step body/title, then displayTitle, then name, then a static placeholder; capped at 200 chars - missing steps: a single Execute-the-fix step generated from the summary; only throws when nothing at all can be derived - missing name: still rejected (normaliseDraft already supplies a name fallback on the LLM path, so this only guards direct validator use) Fixes #2143 --- .../core/skill/crystallize.ts | 31 ++++++++-- .../unit/skill/crystallize-validator.test.ts | 62 +++++++++++++++++++ .../tests/unit/skill/crystallize.test.ts | 11 +++- 3 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 apps/memos-local-plugin/tests/unit/skill/crystallize-validator.test.ts diff --git a/apps/memos-local-plugin/core/skill/crystallize.ts b/apps/memos-local-plugin/core/skill/crystallize.ts index c45c5453d..17e9ad146 100644 --- a/apps/memos-local-plugin/core/skill/crystallize.ts +++ b/apps/memos-local-plugin/core/skill/crystallize.ts @@ -470,11 +470,34 @@ function capString(s: string, cap: number): string { /** * A sensible default validator used both in production and in tests. - * Throws if the draft is structurally unusable (no name, no steps, no summary). + * Throws only when the draft is structurally unusable (no name). Missing + * summary/steps are repaired from the remaining fields instead — LLMs (e.g. + * deepseek-v4-flash) routinely return valid JSON drafts that omit `summary` + * or the `steps` array, and rejecting those stalls the crystallizer queue + * (see issue #2143). */ export function defaultDraftValidator(draft: SkillCrystallizationDraft): void { if (!draft.name) throw new Error("skill.crystallize.invalid: missing name"); - if (!draft.summary) throw new Error("skill.crystallize.invalid: missing summary"); - if (draft.steps.length === 0) - throw new Error("skill.crystallize.invalid: missing steps"); + if (!draft.summary) { + // Auto-generate a summary from the richest available field. Use `||` not + // `??`: LLM JSON emits empty strings, and `??` only falls through on + // null/undefined. + const autoSummary = + draft.steps?.[0]?.body || + draft.steps?.[0]?.title || + draft.displayTitle || + draft.name || + "skill procedure"; + draft.summary = autoSummary.slice(0, 200); + } + if (!draft.steps || draft.steps.length === 0) { + // Auto-generate a single step when the LLM omits the steps array + // (mirror of the summary fallback chain above). + const autoBody = draft.summary || draft.displayTitle || draft.name || ""; + if (autoBody) { + draft.steps = [{ title: "Execute the fix", body: autoBody.slice(0, 2000) }]; + } else { + throw new Error("skill.crystallize.invalid: missing steps"); + } + } } diff --git a/apps/memos-local-plugin/tests/unit/skill/crystallize-validator.test.ts b/apps/memos-local-plugin/tests/unit/skill/crystallize-validator.test.ts new file mode 100644 index 000000000..348a2ac72 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/skill/crystallize-validator.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; + +import { defaultDraftValidator } from "../../../core/skill/crystallize.js"; +import { makeDraft } from "./_helpers.js"; + +describe("defaultDraftValidator", () => { + it("passes a complete draft through unchanged", () => { + const draft = makeDraft(); + expect(() => defaultDraftValidator(draft)).not.toThrow(); + expect(draft.summary).toBe("Ensure system libs exist before pip install on alpine."); + expect(draft.steps).toHaveLength(3); + }); + + it("never throws for a missing summary (issue #2143)", () => { + const draft = makeDraft({ summary: "" }); + expect(() => defaultDraftValidator(draft)).not.toThrow(); + }); + + it("auto-generates summary from the first step body when omitted", () => { + const draft = makeDraft({ summary: "" }); + defaultDraftValidator(draft); + expect(draft.summary).toBe("inspect the pip error for missing .so names"); + }); + + it("falls back through step title, displayTitle, then name for the summary", () => { + const draft = makeDraft({ summary: "", steps: [] }); + defaultDraftValidator(draft); + expect(draft.summary).toBe("Alpine pip install with system deps"); + }); + + it("caps the auto-generated summary at 200 chars", () => { + const longBody = "x".repeat(500); + const draft = makeDraft({ + summary: "", + steps: [{ title: "t", body: longBody }], + }); + defaultDraftValidator(draft); + expect(draft.summary).toBe("x".repeat(200)); + }); + + it("uses || not ?? — an empty-string summary still triggers the fallback", () => { + const draft = makeDraft({ summary: "", steps: [] }); + defaultDraftValidator(draft); + expect(draft.summary).not.toBe(""); + }); + + it("auto-generates a single step when the steps array is empty", () => { + const draft = makeDraft({ steps: [] }); + defaultDraftValidator(draft); + expect(draft.steps).toEqual([ + { + title: "Execute the fix", + body: "Ensure system libs exist before pip install on alpine.", + }, + ]); + }); + + it("still rejects a draft with no name", () => { + const draft = makeDraft({ name: "" }); + expect(() => defaultDraftValidator(draft)).toThrow(/missing name/); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/skill/crystallize.test.ts b/apps/memos-local-plugin/tests/unit/skill/crystallize.test.ts index a2e0e63eb..bcaccaf67 100644 --- a/apps/memos-local-plugin/tests/unit/skill/crystallize.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/crystallize.test.ts @@ -230,7 +230,10 @@ describe("skill/crystallize", () => { expect(r.modelRefusal?.content).toContain("I cannot process this request"); }); - it("rejects drafts that the validator flags as invalid", async () => { + it("repairs drafts the strict validator would have rejected (issue #2143)", async () => { + // A draft with an empty summary AND no steps used to be rejected with + // skill.crystallize.invalid: missing summary / missing steps. The lenient + // validator auto-generates both, so the same draft now crystallizes. const llm = fakeLlm({ completeJson: { "skill.crystallize": makeDraft({ steps: [], summary: "" }) as unknown, @@ -240,6 +243,10 @@ describe("skill/crystallize", () => { { policy: mkPolicy(), evidence: [mkTrace("tr_1", "x")], namingSpace: [] }, { llm, log, config: makeSkillConfig(), validate: defaultDraftValidator }, ); - expect(r.ok).toBe(false); + expect(r.ok).toBe(true); + if (r.ok) { + expect(r.draft.summary).not.toBe(""); + expect(r.draft.steps.length).toBeGreaterThan(0); + } }); }); From fa4bb36bad994120028c190d5cd773a8de5e5903 Mon Sep 17 00:00:00 2001 From: autodev-bot Date: Sun, 16 Aug 2026 11:05:05 +0800 Subject: [PATCH 20/34] fix(hermes-adapter): pass ensure_ascii=False in handle_tool_call json.dumps (#2255) memos_search / memos_get / memos_timeline / memos_skill_list / memos_environment / memos_skill_get in the Hermes memory provider serialized their tool results back to the host LLM with the default ensure_ascii=True, which escaped every non-ASCII code point (notably Chinese memory content) to \uXXXX. The DB stored the correct UTF-8; only the wire JSON was mangled. This inflated tokens for Chinese users and made retrieval results unreadable when debugging. Add ensure_ascii=False to every json.dumps inside handle_tool_call (both the tool-result branches called out in the issue and the error/fallthrough branches, for a uniform pattern that matches the other json.dumps calls in this file at L742/1015/1021 that already pass the flag). Add HandleToolCallEnsureAsciiTests to apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py with 8 regression cases (one per affected tool + both branches of memos_environment) that assert the returned JSON contains raw Chinese characters, contains no "\u" escape marker, and round-trips through json.loads. Verified the tests fail without the fix and pass with it; ruff check + ruff format clean; full test file 35/35 green. Fixes #2255 --- .../hermes/memos_provider/__init__.py | 38 ++- .../python/test_hermes_provider_pipeline.py | 262 ++++++++++++++++++ 2 files changed, 285 insertions(+), 15 deletions(-) diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index e2389e498..244fe027c 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -1504,12 +1504,12 @@ def get_tool_schemas(self) -> list[dict[str, Any]]: # type: ignore[override] def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) -> str: # type: ignore[override] if not self._bridge: - return json.dumps({"error": "bridge not connected"}) + return json.dumps({"error": "bridge not connected"}, ensure_ascii=False) try: if tool_name == "memos_search": query = (args.get("query") or "").strip() if not query: - return json.dumps({"error": "missing query"}) + return json.dumps({"error": "missing query"}, ensure_ascii=False) max_results = self._int_arg(args, "maxResults", 10, 1, 50) params: dict[str, Any] = { "agent": "hermes", @@ -1528,11 +1528,11 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) params, timeout=_LONG_RPC_TIMEOUT, ) - return json.dumps({"hits": resp.get("hits", [])}) + return json.dumps({"hits": resp.get("hits", [])}, ensure_ascii=False) if tool_name == "memos_get": item_id = (args.get("id") or "").strip() if not item_id: - return json.dumps({"error": "missing id"}) + return json.dumps({"error": "missing id"}, ensure_ascii=False) kind = args.get("kind") or "trace" methods = { "trace": "memory.get_trace", @@ -1541,12 +1541,14 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) } method = methods.get(kind) if method is None: - return json.dumps({"error": f"unknown memory kind: {kind}"}) + return json.dumps({"error": f"unknown memory kind: {kind}"}, ensure_ascii=False) item = self._bridge_request_with_retry( method, {"id": item_id, "namespace": self._runtime_namespace()} ) if not item: - return json.dumps({"found": False, "kind": kind, "id": item_id}) + return json.dumps( + {"found": False, "kind": kind, "id": item_id}, ensure_ascii=False + ) if kind == "trace": body = self._clip(item.get("agentText") or item.get("body")) meta = { @@ -1584,7 +1586,8 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) "id": item.get("id", item_id), "body": body, "meta": meta, - } + }, + ensure_ascii=False, ) if tool_name == "memos_timeline": resp = self._bridge_request_with_retry( @@ -1596,13 +1599,16 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) ) limit = self._int_arg(args, "limit", 20, 1, 100) traces = resp.get("traces", [])[:limit] - return json.dumps({"traces": traces}) + return json.dumps({"traces": traces}, ensure_ascii=False) if tool_name == "memos_skill_list": limit = self._int_arg(args, "limit", 10, 1, 50) params = {"limit": limit, "namespace": self._runtime_namespace()} if args.get("status"): params["status"] = args["status"] - return json.dumps(self._bridge_request_with_retry("skill.list", params)) + return json.dumps( + self._bridge_request_with_retry("skill.list", params), + ensure_ascii=False, + ) if tool_name == "memos_environment": query = (args.get("query") or "").strip() limit = self._int_arg(args, "limit", 5, 1, 30) @@ -1621,7 +1627,8 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) for w in resp.get("worldModels", []) ], "queried": False, - } + }, + ensure_ascii=False, ) resp = self._bridge_request_with_retry( "memory.search", @@ -1651,12 +1658,13 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) for h in hits[:limit] ], "queried": True, - } + }, + ensure_ascii=False, ) if tool_name == "memos_skill_get": skill_id = (args.get("id") or "").strip() if not skill_id: - return json.dumps({"error": "missing id"}) + return json.dumps({"error": "missing id"}, ensure_ascii=False) skill = self._bridge_request_with_retry( "skill.get", { @@ -1667,10 +1675,10 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) "episodeId": self._episode_id or None, }, ) - return json.dumps({"found": bool(skill), "skill": skill}) + return json.dumps({"found": bool(skill), "skill": skill}, ensure_ascii=False) except Exception as err: - return json.dumps({"error": str(err)}) - return json.dumps({"error": f"unknown tool: {tool_name}"}) + return json.dumps({"error": str(err)}, ensure_ascii=False) + return json.dumps({"error": f"unknown tool: {tool_name}"}, ensure_ascii=False) # ─── Config schema (for `hermes memory setup`) ──────────────────────── diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index ecd0e1ae1..e85677b00 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -931,5 +931,267 @@ def test_post_llm_call_orders_backfilled_tools_before_later_tool_results(self) - self.assertEqual(turn_end["toolCalls"][1]["thinkingBefore"], "先列计划，再查机票。") +class ChineseToolResultBridge(FakeBridge): + """Fake bridge whose read-path responses embed Chinese characters. + + Used by ``HandleToolCallEnsureAsciiTests`` to prove that every + ``json.dumps`` inside ``handle_tool_call`` passes + ``ensure_ascii=False`` so Chinese memory content is returned to the + host LLM as readable UTF-8 rather than ``\\uXXXX`` escapes (#2255). + """ + + _CH_REFLECTION = "用户提供了 Tushare KEY，需要在后续查询中携带。" + _CH_SNIPPET = "记忆命中：北京晚高峰的地铁调度策略。" + _CH_POLICY_TITLE = "策略：夜间批任务错峰" + _CH_POLICY_BODY = "在凌晨 02:00 之后触发全量导入，避开在线读写高峰。" + _CH_WORLD_TITLE = "世界模型：城市晚高峰" + _CH_WORLD_BODY = "工作日 17:30-19:30 主干道车流密集，通勤需绕行。" + _CH_SKILL_TITLE = "技能：中文摘要" + _CH_SKILL_PROCEDURE = "先分段抽取关键句，再融合成 3 句摘要。" + + def request(self, method: str, params: dict | None = None, **_kwargs: object) -> dict: + payload = params or {} + self.calls.append((method, payload)) + if method == "session.open": + return {"sessionId": payload.get("sessionId") or "hermes:test-session"} + if method == "core.health": + return {"ok": True} + if method == "memory.search": + return { + "hits": [ + { + "id": "trace-cn-1", + "refId": "trace-cn-1", + "refKind": "trace", + "tier": 1, + "score": 0.87, + "snippet": self._CH_SNIPPET, + "reflection": self._CH_REFLECTION, + }, + { + "id": "world-cn-1", + "refId": "world-cn-1", + "refKind": "world_model", + "tier": 3, + "score": 0.71, + "snippet": f"{self._CH_WORLD_TITLE}\n{self._CH_WORLD_BODY}", + }, + ] + } + if method == "memory.get_trace": + return { + "id": payload.get("id"), + "episodeId": "ep-cn-1", + "agentText": "已按用户请求完成中文摘要生成。", + "userText": "帮我把上面的中文材料压缩成 3 句摘要。", + "reflection": self._CH_REFLECTION, + "ts": "2026-08-16T02:50:22Z", + "toolCalls": [], + "value": 0.9, + } + if method == "memory.get_policy": + return { + "id": payload.get("id"), + "title": self._CH_POLICY_TITLE, + "procedure": self._CH_POLICY_BODY, + "trigger": "夜间空闲窗口", + "verification": "首屏读取 P95 无回退", + "boundary": "仅离线批任务", + "gain": "峰值 QPS 下降 30%", + "support": 12, + "status": "active", + } + if method == "memory.get_world": + return { + "id": payload.get("id"), + "title": self._CH_WORLD_TITLE, + "body": self._CH_WORLD_BODY, + "policyIds": ["policy-cn-1"], + } + if method == "memory.timeline": + return { + "traces": [ + { + "id": "trace-cn-1", + "snippet": self._CH_SNIPPET, + "reflection": self._CH_REFLECTION, + } + ] + } + if method == "memory.list_world_models": + return { + "worldModels": [ + { + "id": "world-cn-1", + "title": self._CH_WORLD_TITLE, + "body": self._CH_WORLD_BODY, + "policyIds": ["policy-cn-1"], + } + ] + } + if method == "skill.list": + return { + "skills": [ + { + "id": "skill-cn-1", + "title": self._CH_SKILL_TITLE, + "summary": self._CH_SKILL_PROCEDURE, + } + ] + } + if method == "skill.get": + return { + "id": payload.get("id"), + "title": self._CH_SKILL_TITLE, + "procedure": self._CH_SKILL_PROCEDURE, + } + if method in {"episode.close", "session.close", "subagent.record"}: + return {"ok": True} + raise AssertionError(f"unexpected bridge method: {method}") + + +class HandleToolCallEnsureAsciiTests(unittest.TestCase): + """Regression guard for #2255. + + Every ``json.dumps`` in ``MemTensorProvider.handle_tool_call`` must + pass ``ensure_ascii=False`` so Chinese (and other non-ASCII) memory + content reaches the host LLM as readable UTF-8. Without the flag, + Python's default serialization escapes each non-ASCII code point to + ``\\uXXXX``, dramatically increasing token load and making tool + results unreadable for humans debugging the Hermes side. + """ + + def setUp(self) -> None: + memos_provider.SHARED_BRIDGE_REGISTRY.close_all() + self._mode_patch = patch.dict( + "os.environ", + {"MEMOS_HERMES_BRIDGE_MODE": "legacy"}, + ) + self._mode_patch.start() + + def tearDown(self) -> None: + memos_provider.SHARED_BRIDGE_REGISTRY.close_all() + self._mode_patch.stop() + + def _make_provider(self, bridge: ChineseToolResultBridge) -> object: + patches = ( + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.MemosBridgeClient", return_value=bridge), + ) + for p in patches: + self.addCleanup(p.stop) + p.start() + provider = memos_provider.MemTensorProvider() + provider.initialize( + "hermes:2255", + hermes_home="/tmp/hermes-2255-home", + platform="cli", + agent_identity="hermes-2255", + ) + self.addCleanup(provider.shutdown) + return provider + + # -- individual tools ------------------------------------------------- + + def test_memos_search_returns_utf8_chinese(self) -> None: + bridge = ChineseToolResultBridge() + provider = self._make_provider(bridge) + + raw = provider.handle_tool_call("memos_search", {"query": "中文摘要"}) + + self.assertNotIn("\\u", raw, "memos_search must not \\uXXXX-escape Chinese") + self.assertIn(ChineseToolResultBridge._CH_REFLECTION, raw) + parsed = json.loads(raw) + self.assertEqual( + parsed["hits"][0]["reflection"], + ChineseToolResultBridge._CH_REFLECTION, + ) + + def test_memos_get_trace_returns_utf8_chinese(self) -> None: + bridge = ChineseToolResultBridge() + provider = self._make_provider(bridge) + + raw = provider.handle_tool_call("memos_get", {"id": "trace-cn-1", "kind": "trace"}) + + self.assertNotIn("\\u", raw) + self.assertIn(ChineseToolResultBridge._CH_REFLECTION, raw) + parsed = json.loads(raw) + self.assertTrue(parsed["found"]) + self.assertEqual(parsed["meta"]["reflection"], ChineseToolResultBridge._CH_REFLECTION) + + def test_memos_get_policy_returns_utf8_chinese(self) -> None: + bridge = ChineseToolResultBridge() + provider = self._make_provider(bridge) + + raw = provider.handle_tool_call("memos_get", {"id": "policy-cn-1", "kind": "policy"}) + + self.assertNotIn("\\u", raw) + self.assertIn(ChineseToolResultBridge._CH_POLICY_TITLE, raw) + parsed = json.loads(raw) + self.assertIn(ChineseToolResultBridge._CH_POLICY_BODY, parsed["body"]) + + def test_memos_timeline_returns_utf8_chinese(self) -> None: + bridge = ChineseToolResultBridge() + provider = self._make_provider(bridge) + + raw = provider.handle_tool_call("memos_timeline", {"episodeId": "ep-cn-1"}) + + self.assertNotIn("\\u", raw) + self.assertIn(ChineseToolResultBridge._CH_SNIPPET, raw) + parsed = json.loads(raw) + self.assertEqual(parsed["traces"][0]["snippet"], ChineseToolResultBridge._CH_SNIPPET) + + def test_memos_skill_list_returns_utf8_chinese(self) -> None: + bridge = ChineseToolResultBridge() + provider = self._make_provider(bridge) + + raw = provider.handle_tool_call("memos_skill_list", {"limit": 5}) + + self.assertNotIn("\\u", raw) + self.assertIn(ChineseToolResultBridge._CH_SKILL_TITLE, raw) + + def test_memos_environment_list_returns_utf8_chinese(self) -> None: + bridge = ChineseToolResultBridge() + provider = self._make_provider(bridge) + + raw = provider.handle_tool_call("memos_environment", {"limit": 5}) + + self.assertNotIn("\\u", raw) + self.assertIn(ChineseToolResultBridge._CH_WORLD_BODY, raw) + parsed = json.loads(raw) + self.assertFalse(parsed["queried"]) + self.assertEqual( + parsed["worldModels"][0]["title"], + ChineseToolResultBridge._CH_WORLD_TITLE, + ) + + def test_memos_environment_query_returns_utf8_chinese(self) -> None: + bridge = ChineseToolResultBridge() + provider = self._make_provider(bridge) + + raw = provider.handle_tool_call("memos_environment", {"query": "晚高峰", "limit": 5}) + + self.assertNotIn("\\u", raw) + parsed = json.loads(raw) + self.assertTrue(parsed["queried"]) + self.assertIn(ChineseToolResultBridge._CH_WORLD_TITLE, raw) + + def test_memos_skill_get_returns_utf8_chinese(self) -> None: + bridge = ChineseToolResultBridge() + provider = self._make_provider(bridge) + + raw = provider.handle_tool_call("memos_skill_get", {"id": "skill-cn-1"}) + + self.assertNotIn("\\u", raw) + self.assertIn(ChineseToolResultBridge._CH_SKILL_PROCEDURE, raw) + parsed = json.loads(raw) + self.assertTrue(parsed["found"]) + self.assertEqual( + parsed["skill"]["procedure"], + ChineseToolResultBridge._CH_SKILL_PROCEDURE, + ) + + if __name__ == "__main__": unittest.main() From 492bc8448c99fd0d444d8f2540a3b682a6454502 Mon Sep 17 00:00:00 2001 From: autodev-bot Date: Sun, 16 Aug 2026 11:15:16 +0800 Subject: [PATCH 21/34] test(hermes-adapter): add world_model coverage and drop fragile \u check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address PR #2256 OCR review findings: 1. Add `test_memos_get_world_model_returns_utf8_chinese` to cover the `world_model` branch of `memos_get` (routed via `memory.get_world`). Without this test, an accidental removal of `ensure_ascii=False` from the world_model branch would go undetected — the previous tests only exercised the `trace` and `policy` kinds. 2. Remove the fragile `assertNotIn("\\u", raw)` guard from all HandleToolCallEnsureAsciiTests cases. That check was prone to false positives (any legitimate value containing a backslash followed by `u`, e.g. a Windows path, would fail the test) while providing weaker coverage than the `assertIn(_CH_..., raw)` + `json.loads` field equality assertions already present. Add a header comment explaining why the literal-in-raw check is the robust regression guard. Also add a parsed field-equality assertion to `test_memos_skill_list_returns_utf8_chinese` for parity with the other cases. All 36 tests in tests/python/test_hermes_provider_pipeline.py pass. --- .../python/test_hermes_provider_pipeline.py | 48 +++++++++++++++---- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index e85677b00..73359cf77 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -1093,6 +1093,16 @@ def _make_provider(self, bridge: ChineseToolResultBridge) -> object: return provider # -- individual tools ------------------------------------------------- + # + # Each test proves ``ensure_ascii=False`` by asserting that the raw + # serialized string contains the Chinese literal verbatim. That + # guarantee is stronger than searching for the two-character sequence + # ``\\u`` in the output: with ``ensure_ascii=True`` Python would emit + # ``\uXXXX`` escapes and the raw Chinese literal would NOT appear, so + # ``assertIn(_CH_..., raw)`` alone catches the regression while + # avoiding false positives on legitimate values that just happen to + # contain a backslash followed by ``u`` (e.g. Windows paths, regex + # patterns, or unrelated escape sequences in future fields). def test_memos_search_returns_utf8_chinese(self) -> None: bridge = ChineseToolResultBridge() @@ -1100,7 +1110,6 @@ def test_memos_search_returns_utf8_chinese(self) -> None: raw = provider.handle_tool_call("memos_search", {"query": "中文摘要"}) - self.assertNotIn("\\u", raw, "memos_search must not \\uXXXX-escape Chinese") self.assertIn(ChineseToolResultBridge._CH_REFLECTION, raw) parsed = json.loads(raw) self.assertEqual( @@ -1114,7 +1123,6 @@ def test_memos_get_trace_returns_utf8_chinese(self) -> None: raw = provider.handle_tool_call("memos_get", {"id": "trace-cn-1", "kind": "trace"}) - self.assertNotIn("\\u", raw) self.assertIn(ChineseToolResultBridge._CH_REFLECTION, raw) parsed = json.loads(raw) self.assertTrue(parsed["found"]) @@ -1126,18 +1134,39 @@ def test_memos_get_policy_returns_utf8_chinese(self) -> None: raw = provider.handle_tool_call("memos_get", {"id": "policy-cn-1", "kind": "policy"}) - self.assertNotIn("\\u", raw) self.assertIn(ChineseToolResultBridge._CH_POLICY_TITLE, raw) parsed = json.loads(raw) self.assertIn(ChineseToolResultBridge._CH_POLICY_BODY, parsed["body"]) + def test_memos_get_world_model_returns_utf8_chinese(self) -> None: + """Regression guard for the ``world_model`` branch of ``memos_get``. + + Without this case a future accidental removal of + ``ensure_ascii=False`` from the ``world_model`` branch of + ``memos_get`` (routed via ``memory.get_world``) would go + undetected — the other ``memos_get`` tests only exercise the + ``trace`` and ``policy`` kinds. + """ + bridge = ChineseToolResultBridge() + provider = self._make_provider(bridge) + + raw = provider.handle_tool_call( + "memos_get", {"id": "world-cn-1", "kind": "world_model"} + ) + + self.assertIn(ChineseToolResultBridge._CH_WORLD_TITLE, raw) + parsed = json.loads(raw) + self.assertTrue(parsed["found"]) + self.assertEqual(parsed["kind"], "world_model") + self.assertEqual(parsed["meta"]["title"], ChineseToolResultBridge._CH_WORLD_TITLE) + self.assertIn(ChineseToolResultBridge._CH_WORLD_BODY, parsed["body"]) + def test_memos_timeline_returns_utf8_chinese(self) -> None: bridge = ChineseToolResultBridge() provider = self._make_provider(bridge) raw = provider.handle_tool_call("memos_timeline", {"episodeId": "ep-cn-1"}) - self.assertNotIn("\\u", raw) self.assertIn(ChineseToolResultBridge._CH_SNIPPET, raw) parsed = json.loads(raw) self.assertEqual(parsed["traces"][0]["snippet"], ChineseToolResultBridge._CH_SNIPPET) @@ -1148,8 +1177,12 @@ def test_memos_skill_list_returns_utf8_chinese(self) -> None: raw = provider.handle_tool_call("memos_skill_list", {"limit": 5}) - self.assertNotIn("\\u", raw) self.assertIn(ChineseToolResultBridge._CH_SKILL_TITLE, raw) + parsed = json.loads(raw) + self.assertEqual( + parsed["skills"][0]["title"], + ChineseToolResultBridge._CH_SKILL_TITLE, + ) def test_memos_environment_list_returns_utf8_chinese(self) -> None: bridge = ChineseToolResultBridge() @@ -1157,7 +1190,6 @@ def test_memos_environment_list_returns_utf8_chinese(self) -> None: raw = provider.handle_tool_call("memos_environment", {"limit": 5}) - self.assertNotIn("\\u", raw) self.assertIn(ChineseToolResultBridge._CH_WORLD_BODY, raw) parsed = json.loads(raw) self.assertFalse(parsed["queried"]) @@ -1172,10 +1204,9 @@ def test_memos_environment_query_returns_utf8_chinese(self) -> None: raw = provider.handle_tool_call("memos_environment", {"query": "晚高峰", "limit": 5}) - self.assertNotIn("\\u", raw) + self.assertIn(ChineseToolResultBridge._CH_WORLD_TITLE, raw) parsed = json.loads(raw) self.assertTrue(parsed["queried"]) - self.assertIn(ChineseToolResultBridge._CH_WORLD_TITLE, raw) def test_memos_skill_get_returns_utf8_chinese(self) -> None: bridge = ChineseToolResultBridge() @@ -1183,7 +1214,6 @@ def test_memos_skill_get_returns_utf8_chinese(self) -> None: raw = provider.handle_tool_call("memos_skill_get", {"id": "skill-cn-1"}) - self.assertNotIn("\\u", raw) self.assertIn(ChineseToolResultBridge._CH_SKILL_PROCEDURE, raw) parsed = json.loads(raw) self.assertTrue(parsed["found"]) From 117b2a2d1414a09d1279ebf6f9d3585d509ceb87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Mon, 24 Aug 2026 19:13:24 +0800 Subject: [PATCH 22/34] fix(plugin): recover gateway after installer failures --- apps/memos-local-plugin/install.ps1 | 96 ++++++-- apps/memos-local-plugin/install.sh | 30 ++- .../install/install-gateway-recovery.test.ts | 225 ++++++++++++++++++ .../tests/unit/install/install-ps1.test.ts | 53 +++++ 4 files changed, 379 insertions(+), 25 deletions(-) create mode 100644 apps/memos-local-plugin/tests/unit/install/install-gateway-recovery.test.ts diff --git a/apps/memos-local-plugin/install.ps1 b/apps/memos-local-plugin/install.ps1 index 7afbabb65..be5fc85c7 100644 --- a/apps/memos-local-plugin/install.ps1 +++ b/apps/memos-local-plugin/install.ps1 @@ -48,6 +48,30 @@ function Invoke-NativeChecked { } } +function Invoke-OpenClawGatewayChecked { + param( + [ValidateSet("start", "stop")] + [string]$Action + ) + # PowerShell 5.1 can promote a native process' stderr to an ErrorRecord. + # Capture it without turning it into a terminating PowerShell error, then + # use the native exit code as the authoritative result. + $PreviousErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = "Continue" + try { + $GatewayOutput = @(& cmd.exe /d /c "openclaw gateway $Action" 2>&1) + $ExitCode = $LASTEXITCODE + } finally { + $ErrorActionPreference = $PreviousErrorActionPreference + } + foreach ($Line in $GatewayOutput) { + Write-Host "$Line" + } + if ($ExitCode -ne 0) { + throw "openclaw gateway $Action failed (exit code $ExitCode)" + } +} + function Test-BetterSqlite3 { param([string]$NodeBin, [string]$Prefix) $SmokeScript = "const Database=require('better-sqlite3');const db=new Database(':memory:');db.exec('SELECT 1');db.close();" @@ -157,6 +181,7 @@ if ($AgentSelection -eq "auto") { # Resolve tarball $StageDir = New-Item -ItemType Directory -Path (Join-Path $env:TEMP ([guid]::NewGuid().ToString())) -Force +try { $SourceKind = "npm" $SourceSpec = $NpmPackage $BuiltTarball = "" @@ -482,18 +507,21 @@ function Install-OpenClaw { $ConfigPath = Join-Path $env:USERPROFILE ".openclaw\openclaw.json" $OcBin = Get-Command "openclaw" -ErrorAction SilentlyContinue - if ($OcBin) { - Write-Info "Stopping OpenClaw gateway" - cmd /c "openclaw gateway stop" - Start-Sleep -Seconds 1 - } + $GatewayRecoveryState = "inactive" + try { + if ($OcBin) { + Write-Info "Stopping OpenClaw gateway" + Invoke-OpenClawGatewayChecked -Action "stop" + $GatewayRecoveryState = "needs_recovery" + Start-Sleep -Seconds 1 + } - Deploy-Tarball -Prefix $Prefix + Deploy-Tarball -Prefix $Prefix - $RuntimeEntry = "./dist/adapters/openclaw/index.js" - if (-not (Test-Path (Join-Path $Prefix "dist\adapters\openclaw\index.js"))) { - Stop-Die "OpenClaw runtime entry missing." - } + $RuntimeEntry = "./dist/adapters/openclaw/index.js" + if (-not (Test-Path (Join-Path $Prefix "dist\adapters\openclaw\index.js"))) { + throw "OpenClaw runtime entry missing." + } Ensure-RuntimeHome -Agent "openclaw" -HomeDir $HomeDir -Prefix $Prefix @@ -628,21 +656,40 @@ config.plugins.entries[pluginId].hooks.allowConversationAccess = true; fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n', 'utf8'); "@ - $NodeScriptPath = Join-Path $env:TEMP "patch_openclaw.js" - Set-Content -Path $NodeScriptPath -Value $NodeScript -Encoding UTF8 - node $NodeScriptPath - Write-Success "openclaw.json patched" + $NodeScriptPath = Join-Path $env:TEMP "patch_openclaw.js" + Set-Content -Path $NodeScriptPath -Value $NodeScript -Encoding UTF8 + Invoke-NativeChecked -Command "node" -Arguments @($NodeScriptPath) -FailureMessage "Failed to patch openclaw.json" + Write-Success "openclaw.json patched" - if ($OcBin) { - Write-Info "Starting OpenClaw gateway" - cmd /c "openclaw gateway start" - if (Wait-ForViewer -Port $OpenClawPort) { - Write-Success "OpenClaw install complete" + if ($OcBin) { + Write-Info "Starting OpenClaw gateway" + try { + Invoke-OpenClawGatewayChecked -Action "start" + } catch { + # The regular final start already ran. Do not invoke it a + # second time from recovery cleanup and hide the real failure. + $GatewayRecoveryState = "final_failed" + throw + } + $GatewayRecoveryState = "inactive" + if (Wait-ForViewer -Port $OpenClawPort) { + Write-Success "OpenClaw install complete" + } else { + Write-Warn "Memory Viewer did not respond." + } } else { - Write-Warn "Memory Viewer did not respond." + Write-Warn "openclaw CLI not found. Start gateway manually." + } + } finally { + if ($GatewayRecoveryState -eq "needs_recovery") { + Write-Warn "Install failed after stopping OpenClaw; restarting the gateway." + try { + Invoke-OpenClawGatewayChecked -Action "start" + Write-Success "OpenClaw gateway recovered" + } catch { + Write-Warn "OpenClaw gateway recovery failed: $($_.Exception.Message)" + } } - } else { - Write-Warn "openclaw CLI not found. Start gateway manually." } } @@ -786,3 +833,8 @@ if ($AgentSelection -eq "hermes" -or $AgentSelection -eq "all") { Install-Hermes Write-Host "`n ==================================================" -ForegroundColor Green Write-Host " Install finished successfully! " -ForegroundColor Green Write-Host " ==================================================`n" -ForegroundColor Green +} finally { + if ($StageDir -and (Test-Path $StageDir)) { + Remove-Item -Recurse -Force $StageDir -ErrorAction SilentlyContinue + } +} diff --git a/apps/memos-local-plugin/install.sh b/apps/memos-local-plugin/install.sh index bd972c8f1..5f2c8184e 100755 --- a/apps/memos-local-plugin/install.sh +++ b/apps/memos-local-plugin/install.sh @@ -299,8 +299,20 @@ STAGE_DIR="" DSH_PNPM_TEMP_DIR="" SOURCE_KIND="" # "path" for a local file, "npm" otherwise SOURCE_SPEC="" - -cleanup_install_temp_dirs() { +GATEWAY_RECOVERY_BIN="" +GATEWAY_RECOVERY_STATE="inactive" + +cleanup_install_state() { + if [[ "${GATEWAY_RECOVERY_STATE:-inactive}" == "needs_recovery" \ + && -n "${GATEWAY_RECOVERY_BIN:-}" ]]; then + local recovery_out="" + if ! recovery_out="$("${GATEWAY_RECOVERY_BIN}" gateway start 2>&1)"; then + warn "OpenClaw gateway recovery failed; the gateway may still be stopped." + if [[ -n "${recovery_out}" ]]; then + printf '%s\n' "${recovery_out}" | sed 's/^/ /' >&2 + fi + fi + fi if [[ -n "${STAGE_DIR}" && -d "${STAGE_DIR}" ]]; then rm -rf -- "${STAGE_DIR}" fi @@ -308,7 +320,7 @@ cleanup_install_temp_dirs() { rm -rf -- "${DSH_PNPM_TEMP_DIR}" fi } -trap cleanup_install_temp_dirs EXIT +trap cleanup_install_state EXIT resolve_source_spec() { if [[ -n "${VERSION_ARG}" && -f "${VERSION_ARG}" ]]; then @@ -484,11 +496,16 @@ install_openclaw() { mkdir -p "${HOME}/.openclaw" local oc_bin="" + # These remain global because the EXIT trap runs after this function returns. + GATEWAY_RECOVERY_BIN="" + GATEWAY_RECOVERY_STATE="inactive" if oc_bin="$(find_openclaw_cli)"; then step "Stopping OpenClaw gateway" "${oc_bin}" gateway stop >/dev/null 2>&1 || true sleep 1 success "Gateway stopped" + GATEWAY_RECOVERY_BIN="${oc_bin}" + GATEWAY_RECOVERY_STATE="needs_recovery" fi deploy_tarball_to_prefix "${prefix}" @@ -663,6 +680,9 @@ NODE || (command -v lsof >/dev/null 2>&1 && lsof -i ":18789" -t >/dev/null 2>&1); then success "OpenClaw gateway already running" else + # The intended final start already ran and failed; do not repeat the same + # command from the EXIT trap. + GATEWAY_RECOVERY_STATE="final_failed" error "openclaw gateway start failed:" echo "${start_out}" | sed 's/^/ /' >&2 warn "Inspect ~/.openclaw/logs/gateway.err.log for the full reason." @@ -671,6 +691,10 @@ NODE else success "OpenClaw gateway started" fi + # The service started (or was already running), so later viewer fallback + # failures must not trigger another service start from the EXIT trap. + GATEWAY_RECOVERY_STATE="inactive" + GATEWAY_RECOVERY_BIN="" step "Waiting for Memory Viewer" if wait_for_viewer "${OPENCLAW_PORT}"; then diff --git a/apps/memos-local-plugin/tests/unit/install/install-gateway-recovery.test.ts b/apps/memos-local-plugin/tests/unit/install/install-gateway-recovery.test.ts new file mode 100644 index 000000000..37dfd651d --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/install/install-gateway-recovery.test.ts @@ -0,0 +1,225 @@ +import { spawnSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const REPO_ROOT = path.resolve(__dirname, "..", "..", ".."); +const INSTALLER = path.join(REPO_ROOT, "install.sh"); + +interface InstallerFixture { + root: string; + home: string; + bin: string; + temp: string; + gatewayLog: string; + env: NodeJS.ProcessEnv; +} + +function writeExecutable(file: string, body: string): void { + writeFileSync(file, `#!/usr/bin/env bash\nset -u\n${body}\n`, "utf8"); + chmodSync(file, 0o755); +} + +function createFixture(): InstallerFixture { + const root = mkdtempSync(path.join(tmpdir(), "memos-installer-recovery-")); + const home = path.join(root, "home"); + const bin = path.join(root, "bin"); + const temp = path.join(root, "tmp"); + const gatewayLog = path.join(root, "gateway.log"); + mkdirSync(path.join(home, ".openclaw"), { recursive: true }); + mkdirSync(bin); + mkdirSync(temp); + + writeExecutable( + path.join(bin, "node"), + `if [[ "\${1:-}" == "-v" ]]; then + printf 'v22.0.0\\n' +elif [[ "\${1:-}" == "-p" ]]; then + printf '1.0.0\\n' +fi +exit 0`, + ); + writeExecutable( + path.join(bin, "npm"), + `if [[ "\${1:-}" == "install" ]]; then + mkdir -p node_modules/better-sqlite3 +fi +exit 0`, + ); + writeExecutable( + path.join(bin, "openclaw"), + `printf '%s\\n' "$*" >> "\${FAKE_GATEWAY_LOG:?}" +if [[ "$*" == "gateway start" ]]; then + if [[ "\${FAKE_GATEWAY_START_EXIT:-0}" != "0" ]]; then + printf 'fake gateway start failure\\n' >&2 + fi + exit "\${FAKE_GATEWAY_START_EXIT:-0}" +fi +exit 0`, + ); + writeExecutable(path.join(bin, "sleep"), "exit 0"); + writeExecutable(path.join(bin, "lsof"), "exit 1"); + writeExecutable(path.join(bin, "curl"), 'exit "${FAKE_CURL_EXIT:-0}"'); + + return { + root, + home, + bin, + temp, + gatewayLog, + env: { + ...process.env, + HOME: home, + TMPDIR: temp, + PATH: `${bin}:${process.env.PATH ?? ""}`, + FAKE_GATEWAY_LOG: gatewayLog, + }, + }; +} + +function runInstaller( + fixture: InstallerFixture, + version: string, + extraEnv: NodeJS.ProcessEnv = {}, +) { + return spawnSync( + "bash", + [INSTALLER, "--agent", "openclaw", "--version", version], + { + cwd: fixture.root, + encoding: "utf8", + timeout: 30_000, + env: { ...fixture.env, ...extraEnv }, + }, + ); +} + +function gatewayCalls(fixture: InstallerFixture): string[] { + if (!existsSync(fixture.gatewayLog)) return []; + return readFileSync(fixture.gatewayLog, "utf8") + .trim() + .split("\n") + .filter(Boolean); +} + +function expectTemporaryDirectoriesCleaned(fixture: InstallerFixture): void { + expect(readdirSync(fixture.temp)).toEqual([]); +} + +function createValidPackage(fixture: InstallerFixture): string { + const packageRoot = path.join(fixture.root, "package"); + const runtimeDir = path.join(packageRoot, "dist", "adapters", "openclaw"); + const tarball = path.join(fixture.root, "plugin.tgz"); + mkdirSync(runtimeDir, { recursive: true }); + writeFileSync( + path.join(packageRoot, "package.json"), + '{"name":"test-plugin","version":"1.0.0"}\n', + "utf8", + ); + writeFileSync(path.join(runtimeDir, "index.js"), "export {};\n", "utf8"); + const result = spawnSync( + "tar", + ["-czf", tarball, "-C", fixture.root, "package"], + { encoding: "utf8" }, + ); + expect(result.status, result.stderr).toBe(0); + return tarball; +} + +describe.skipIf(process.platform === "win32")( + "unified installer gateway recovery", + () => { + it("restarts the gateway when package extraction fails after it was stopped", () => { + const fixture = createFixture(); + try { + const brokenTarball = path.join(fixture.root, "broken.tgz"); + writeFileSync(brokenTarball, "not a tarball", "utf8"); + + const result = runInstaller(fixture, brokenTarball); + + expect(result.status).toBe(1); + expect(gatewayCalls(fixture)).toEqual([ + "gateway stop", + "gateway start", + ]); + expectTemporaryDirectoriesCleaned(fixture); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } + }); + + it("reports a failed recovery start without masking the install failure", () => { + const fixture = createFixture(); + try { + const brokenTarball = path.join(fixture.root, "broken.tgz"); + writeFileSync(brokenTarball, "not a tarball", "utf8"); + + const result = runInstaller(fixture, brokenTarball, { + FAKE_GATEWAY_START_EXIT: "17", + }); + + expect(result.status).toBe(1); + expect(gatewayCalls(fixture)).toEqual([ + "gateway stop", + "gateway start", + ]); + expect(result.stderr).toContain("OpenClaw gateway recovery failed"); + expect(result.stderr).toContain("fake gateway start failure"); + expectTemporaryDirectoriesCleaned(fixture); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } + }); + + it("does not retry the normal final gateway start from exit cleanup", () => { + const fixture = createFixture(); + try { + const tarball = createValidPackage(fixture); + + const result = runInstaller(fixture, tarball, { + FAKE_CURL_EXIT: "1", + FAKE_GATEWAY_START_EXIT: "17", + }); + + expect(result.status).not.toBe(0); + expect(gatewayCalls(fixture)).toEqual([ + "gateway stop", + "gateway start", + ]); + expect(result.stderr).toContain("openclaw gateway start failed"); + expectTemporaryDirectoriesCleaned(fixture); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } + }); + + it("disarms recovery after the normal gateway start succeeds", () => { + const fixture = createFixture(); + try { + const tarball = createValidPackage(fixture); + + const result = runInstaller(fixture, tarball); + + expect(result.status).toBe(0); + expect(gatewayCalls(fixture)).toEqual([ + "gateway stop", + "gateway start", + ]); + expectTemporaryDirectoriesCleaned(fixture); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } + }); + }, +); diff --git a/apps/memos-local-plugin/tests/unit/install/install-ps1.test.ts b/apps/memos-local-plugin/tests/unit/install/install-ps1.test.ts index 303754579..166a5b8a4 100644 --- a/apps/memos-local-plugin/tests/unit/install/install-ps1.test.ts +++ b/apps/memos-local-plugin/tests/unit/install/install-ps1.test.ts @@ -14,6 +14,59 @@ function extractOpenClawConfigPatch(script: string): string { return match[1]; } +function extractFunction(script: string, name: string, nextName: string): string { + const start = script.indexOf(`function ${name}`); + const end = script.indexOf(`function ${nextName}`, start); + if (start < 0 || end < 0) throw new Error(`${name} function not found`); + return script.slice(start, end); +} + +describe("install.ps1 — gateway recovery", () => { + it("restarts OpenClaw from finally when installation fails after stop", () => { + const script = readFileSync(SCRIPT, "utf8"); + const installOpenClaw = extractFunction( + script, + "Install-OpenClaw", + "Install-Hermes", + ); + + expect(installOpenClaw).toMatch( + /Invoke-OpenClawGatewayChecked\s+-Action\s+"stop"[\s\S]*\$GatewayRecoveryState\s*=\s*"needs_recovery"/, + ); + expect(installOpenClaw).toMatch( + /finally\s*\{[\s\S]*\$GatewayRecoveryState\s+-eq\s+"needs_recovery"[\s\S]*Invoke-OpenClawGatewayChecked\s+-Action\s+"start"/, + ); + }); + + it("checks native gateway failures and does not retry a failed final start", () => { + const script = readFileSync(SCRIPT, "utf8"); + const installOpenClaw = extractFunction( + script, + "Install-OpenClaw", + "Install-Hermes", + ); + const invokeGateway = extractFunction( + script, + "Invoke-OpenClawGatewayChecked", + "Test-BetterSqlite3", + ); + + expect(invokeGateway).toContain("$ExitCode = $LASTEXITCODE"); + expect(invokeGateway).toMatch(/if \(\$ExitCode -ne 0\)[\s\S]*throw/); + expect(installOpenClaw).toMatch( + /catch\s*\{[\s\S]*\$GatewayRecoveryState\s*=\s*"final_failed"[\s\S]*throw/, + ); + }); + + it("cleans the installer staging directory from a top-level finally block", () => { + const script = readFileSync(SCRIPT, "utf8"); + + expect(script).toMatch( + /\}\s*finally\s*\{\s*if \(\$StageDir -and \(Test-Path \$StageDir\)\) \{\s*Remove-Item[^\n]*\$StageDir[\s\S]*\}\s*\}\s*$/, + ); + }); +}); + describe("install.ps1 — OpenClaw config patch", () => { it("preserves existing hook settings and enables conversation access", () => { const script = readFileSync(SCRIPT, "utf8"); From 060859d0a2bbc1dfab178b463bd7891687576889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Mon, 24 Aug 2026 20:57:15 +0800 Subject: [PATCH 23/34] fix(plugin): run idle skill archival in background --- .../core/pipeline/memory-core.ts | 5 +- .../core/pipeline/orchestrator.ts | 11 ++- .../core/skill/ALGORITHMS.md | 13 ++- apps/memos-local-plugin/core/skill/README.md | 4 + .../core/skill/lifecycle-worker.ts | 82 ++++++++++++++++ .../core/skill/subscriber.ts | 41 ++++---- .../core/storage/repos/skills.ts | 93 +++++++------------ .../tests/unit/pipeline/memory-core.test.ts | 38 ++++++++ .../tests/unit/skill/lifecycle-worker.test.ts | 90 ++++++++++++++++++ .../tests/unit/skill/subscriber.test.ts | 61 ------------ .../tests/unit/storage/repos.test.ts | 49 ++++------ 11 files changed, 304 insertions(+), 183 deletions(-) create mode 100644 apps/memos-local-plugin/core/skill/lifecycle-worker.ts create mode 100644 apps/memos-local-plugin/tests/unit/skill/lifecycle-worker.test.ts diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index 4677ece0c..6d8f120de 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -5026,7 +5026,10 @@ export function createMemoryCore( const existing = handle.repos.skills.getById(id); if (!existing || !ownedByCurrent(existing)) return null; const now = Date.now(); - handle.repos.skills.setStatus(id, "active", now); + handle.db.tx(() => { + handle.repos.skills.setStatus(id, "active", now); + handle.repos.skills.recordUse(id, now); + }); if (existing.status !== "active") { handle.buses.skill.emit({ kind: "skill.status.changed", diff --git a/apps/memos-local-plugin/core/pipeline/orchestrator.ts b/apps/memos-local-plugin/core/pipeline/orchestrator.ts index 9e31467e0..97bdac299 100644 --- a/apps/memos-local-plugin/core/pipeline/orchestrator.ts +++ b/apps/memos-local-plugin/core/pipeline/orchestrator.ts @@ -88,6 +88,7 @@ import { prioritizeEmbedder, } from "../util/foreground-resources.js"; import { createRequestDeadline } from "../util/request-deadline.js"; +import { createSkillLifecycleWorker } from "../skill/lifecycle-worker.js"; function classifyWithTimeout( classifyFn: () => Promise, @@ -307,6 +308,12 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { log, emit: emitCore, }); + const skillLifecycleWorker = createSkillLifecycleWorker({ + runLifecycle: () => subs.skills.lifecycleTick(), + log: log.child({ channel: "core.skill.lifecycle-worker" }), + now: deps.now, + }); + if (!lightweightMode) skillLifecycleWorker.start(); // In-memory index of the open episode per session so we can route // `addTurn` calls without a repo round-trip. @@ -1566,6 +1573,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { toolCalls: result.toolCalls.length, agentChars: result.agentText.length, }); + skillLifecycleWorker.trigger(); // The episode stays OPEN — finalize is deferred to topic end. return { @@ -1638,7 +1646,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { await subs.l3.drain(); await nextTick(); await subs.skills.flush(); - await subs.skills.lifecycleTick(); + await skillLifecycleWorker.runNow(); await subs.feedback.flush(); await embeddingRetryWorker.flush(); } @@ -1649,6 +1657,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { // capture and downstream enrichment. Hermes' bridge owns a 20s outer // shutdown ceiling, so abort before that rather than either hanging or // discarding every single-shot session's enrichment immediately. + skillLifecycleWorker.stop(); embeddingRetryWorker.stop(); const flushPromise = flush(); try { diff --git a/apps/memos-local-plugin/core/skill/ALGORITHMS.md b/apps/memos-local-plugin/core/skill/ALGORITHMS.md index 205c3e7b2..e1560d926 100644 --- a/apps/memos-local-plugin/core/skill/ALGORITHMS.md +++ b/apps/memos-local-plugin/core/skill/ALGORITHMS.md @@ -252,10 +252,15 @@ the next lifecycle tick. `lastUsedAt` is updated by the existing recorded-use path. A never-used skill falls back to `createdAt`; unrelated metadata updates therefore do -not reset its idle clock. The scan runs through the orchestrator's normal -flush lifecycle and does not introduce a separate timer. Each tick processes -at most ten 500-row batches; any remaining backlog is deferred to a later tick -so a large archive queue cannot monopolize the event loop. +not reset its idle clock. Manual reactivation records a fresh use timestamp, +giving the skill a complete grace period before it can be archived again. + +A single-flight lifecycle worker runs once at startup, at most hourly while +the process remains alive, and opportunistically after turn completion. It +does not drain the capture/reward/L2/L3 chain. Each tick atomically selects +and updates at most ten 500-row batches, yielding between full batches; any +remaining backlog is deferred so a large archive queue cannot monopolize the +event loop. --- diff --git a/apps/memos-local-plugin/core/skill/README.md b/apps/memos-local-plugin/core/skill/README.md index a0e573b8c..04a2de8bf 100644 --- a/apps/memos-local-plugin/core/skill/README.md +++ b/apps/memos-local-plugin/core/skill/README.md @@ -210,6 +210,10 @@ See `algorithm.skill` in | `minEtaForRetrieval` | `0.5` | η gate for Tier-1 retrieval + auto-promotion. | | `idleArchiveMs` | `2592000000` | Archive low-η active skills after 30 days without use (minimum 1 hour). | +Idle archival is maintained by a single-flight background worker that runs +at startup and at most hourly. Manually reactivating a Skill starts a fresh +idle grace period. + ## Logging All skill work logs on dedicated channels (see diff --git a/apps/memos-local-plugin/core/skill/lifecycle-worker.ts b/apps/memos-local-plugin/core/skill/lifecycle-worker.ts new file mode 100644 index 000000000..d3daf0f1a --- /dev/null +++ b/apps/memos-local-plugin/core/skill/lifecycle-worker.ts @@ -0,0 +1,82 @@ +import type { Logger } from "../logger/types.js"; + +export const DEFAULT_SKILL_LIFECYCLE_INTERVAL_MS = 60 * 60 * 1000; + +export interface SkillLifecycleWorker { + start(): void; + trigger(): void; + runNow(): Promise; + flush(): Promise; + stop(): void; +} + +export interface SkillLifecycleWorkerDeps { + runLifecycle(): Promise; + log: Logger; + intervalMs?: number; + now?: () => number; +} + +/** + * Periodically runs lightweight Skill lifecycle maintenance without draining + * the full capture/reward/L2/L3 pipeline. Scheduled failures are isolated so + * one bad pass cannot permanently stop future maintenance. + */ +export function createSkillLifecycleWorker( + deps: SkillLifecycleWorkerDeps, +): SkillLifecycleWorker { + const intervalMs = Math.max( + 1, + Math.floor(deps.intervalMs ?? DEFAULT_SKILL_LIFECYCLE_INTERVAL_MS), + ); + const now = deps.now ?? Date.now; + let timer: ReturnType | null = null; + let running: Promise | null = null; + let lastStartedAt = Number.NEGATIVE_INFINITY; + let stopped = true; + + function beginRun(): Promise { + if (running) return running; + lastStartedAt = now(); + const current = Promise.resolve().then(() => deps.runLifecycle()).finally(() => { + if (running === current) running = null; + }); + running = current; + return current; + } + + function trigger(): void { + if (stopped || running || now() - lastStartedAt < intervalMs) return; + void beginRun().catch((err) => { + deps.log.warn("skill.lifecycle_worker.failed", { + err: err instanceof Error ? err.message : String(err), + }); + }); + } + + return { + start(): void { + if (!stopped) return; + stopped = false; + trigger(); + timer = setInterval(trigger, intervalMs); + (timer as unknown as { unref?: () => void }).unref?.(); + }, + + trigger, + + runNow(): Promise { + return beginRun(); + }, + + async flush(): Promise { + if (running) await running; + }, + + stop(): void { + stopped = true; + if (timer) clearInterval(timer); + timer = null; + }, + }; +} diff --git a/apps/memos-local-plugin/core/skill/subscriber.ts b/apps/memos-local-plugin/core/skill/subscriber.ts index 67203b87c..6b30d9d5e 100644 --- a/apps/memos-local-plugin/core/skill/subscriber.ts +++ b/apps/memos-local-plugin/core/skill/subscriber.ts @@ -37,9 +37,13 @@ import type { SkillId } from "../types.js"; import { now as nowMs } from "../time.js"; import { IDLE_ARCHIVE_BATCH_LIMIT } from "../storage/repos/skills.js"; -/** Bound one lifecycle pass to 5,000 archival writes. */ +/** Bound one lifecycle pass to ten repository-sized archival batches. */ const IDLE_ARCHIVE_MAX_BATCHES_PER_TICK = 10; +function yieldToEventLoop(): Promise { + return new Promise((resolve) => setImmediate(resolve)); +} + export interface SkillSubscriberDeps extends Omit { log?: Logger; @@ -236,27 +240,17 @@ export function attachSkillSubscriber( let batchesProcessed = 0; let archivedTotal = 0; while (batchesProcessed < IDLE_ARCHIVE_MAX_BATCHES_PER_TICK) { - const archiveCandidates = deps.repos.skills.listIdleArchiveCandidates({ + const archivedSkills = deps.repos.skills.archiveNextIdleBatch({ minEtaForRetrieval: deps.config.minEtaForRetrieval, cutoff, + updatedAt: at, limit: IDLE_ARCHIVE_BATCH_LIMIT, }); batchesProcessed += 1; - const archivedIds = new Set( - deps.repos.skills.archiveIdleBatch( - archiveCandidates.map((skill) => skill.id), - { - minEtaForRetrieval: deps.config.minEtaForRetrieval, - cutoff, - updatedAt: at, - }, - ), - ); - const archivedThisBatch = archivedIds.size; + const archivedThisBatch = archivedSkills.length; archivedTotal += archivedThisBatch; - for (const s of archiveCandidates) { - if (!archivedIds.has(s.id)) continue; - log.info("skill.idle_archived", { + for (const s of archivedSkills) { + log.debug("skill.idle_archived", { skillId: s.id, name: s.name, eta: s.eta, @@ -272,24 +266,23 @@ export function attachSkillSubscriber( transition: "archived", }); } - if (archiveCandidates.length > 0 && archivedThisBatch === 0) { - // A full zero-change batch was invalidated by concurrent writers. - // Re-query so later eligible rows are not abandoned for this tick. - log.warn("skill.idle_archive_stalled", { - candidateCount: archiveCandidates.length, + if (archivedThisBatch > 0) { + log.info("skill.idle_archive_batch", { + batchCount: batchesProcessed, + archivedCount: archivedThisBatch, cutoff, minEtaForRetrieval: deps.config.minEtaForRetrieval, }); - if (archiveCandidates.length < IDLE_ARCHIVE_BATCH_LIMIT) break; - continue; } - if (archiveCandidates.length < IDLE_ARCHIVE_BATCH_LIMIT) break; + if (archivedThisBatch < IDLE_ARCHIVE_BATCH_LIMIT) break; if (batchesProcessed === IDLE_ARCHIVE_MAX_BATCHES_PER_TICK) { log.warn("skill.idle_archive_batch_limit_reached", { batchCount: batchesProcessed, archivedCount: archivedTotal, batchSize: IDLE_ARCHIVE_BATCH_LIMIT, }); + } else { + await yieldToEventLoop(); } } } diff --git a/apps/memos-local-plugin/core/storage/repos/skills.ts b/apps/memos-local-plugin/core/storage/repos/skills.ts index 7eb493dda..8a04d8e70 100644 --- a/apps/memos-local-plugin/core/storage/repos/skills.ts +++ b/apps/memos-local-plugin/core/storage/repos/skills.ts @@ -66,17 +66,6 @@ export function makeSkillsRepo(db: StorageDb) { const updateStatus = db.prepare( buildUpdate({ table: "skills", columns: ["id", "status", "updated_at"] }), ); - const archiveIdle = db.prepare<{ - id: string; - min_eta: number; - cutoff: number; - updated_at: number; - }>( - `UPDATE skills - SET status = 'archived', updated_at = @updated_at - WHERE id = @id - AND ${IDLE_ARCHIVE_PREDICATE}`, - ); const updateTrials = db.prepare( buildUpdate({ table: "skills", @@ -103,24 +92,40 @@ export function makeSkillsRepo(db: StorageDb) { updateStatus.run({ id, status, updated_at: updatedAt }); }, - archiveIdleBatch( - ids: readonly SkillId[], - input: { minEtaForRetrieval: number; cutoff: number; updatedAt: number }, - ): SkillId[] { - if (ids.length === 0) return []; - return db.tx(() => { - const archived: SkillId[] = []; - for (const id of ids) { - const res = archiveIdle.run({ - id, - min_eta: input.minEtaForRetrieval, - cutoff: input.cutoff, - updated_at: input.updatedAt, - }); - if (res.changes > 0) archived.push(id); - } - return archived; - }); + /** + * Atomically select and archive one oldest-first batch. Keeping candidate + * selection and the conditional transition in one SQLite statement + * removes the read/update race and avoids hundreds of UPDATE round-trips. + */ + archiveNextIdleBatch(input: { + minEtaForRetrieval: number; + cutoff: number; + updatedAt: number; + limit?: number; + }): SkillRow[] { + const requestedLimit = Number.isFinite(input.limit) + ? Math.floor(input.limit!) + : IDLE_ARCHIVE_BATCH_LIMIT; + const params = { + min_eta: input.minEtaForRetrieval, + cutoff: input.cutoff, + updated_at: input.updatedAt, + limit: Math.max(1, Math.min(IDLE_ARCHIVE_BATCH_LIMIT, requestedLimit)), + }; + const sql = ` + WITH candidates AS ( + SELECT id + FROM skills + WHERE ${IDLE_ARCHIVE_PREDICATE} + ORDER BY COALESCE(last_used_at, created_at) ASC + LIMIT @limit + ) + UPDATE skills + SET status = 'archived', updated_at = @updated_at + WHERE id IN (SELECT id FROM candidates) + AND ${IDLE_ARCHIVE_PREDICATE} + RETURNING ${COLUMNS.join(", ")}`; + return db.prepare(sql).all(params).map(mapRow); }, bumpTrial( @@ -176,36 +181,6 @@ export function makeSkillsRepo(db: StorageDb) { return db.prepare(sql).all(params).map(mapRow); }, - /** - * Return one oldest-first batch of active skills that already satisfy - * the idle-archive predicate. Filtering in SQLite prevents unrelated - * recently-updated skills from starving older candidates. - */ - listIdleArchiveCandidates(input: { - minEtaForRetrieval: number; - cutoff: number; - limit?: number; - }): SkillRow[] { - const params = { - min_eta: input.minEtaForRetrieval, - cutoff: input.cutoff, - limit: Math.max( - 1, - Math.min( - IDLE_ARCHIVE_BATCH_LIMIT, - Math.floor(input.limit ?? IDLE_ARCHIVE_BATCH_LIMIT), - ), - ), - }; - const sql = ` - SELECT ${COLUMNS.join(", ")} - FROM skills - WHERE ${IDLE_ARCHIVE_PREDICATE} - ORDER BY COALESCE(last_used_at, created_at) ASC - LIMIT @limit`; - return db.prepare(sql).all(params).map(mapRow); - }, - count(filter: Omit = {}): number { const fragments: string[] = []; const params: Record = {}; diff --git a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts index b4c095079..7e2a7fb4f 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts @@ -1531,6 +1531,44 @@ describe("MemoryCore façade", () => { name: "local skill", }); }); + + it("gives a manually reactivated skill a fresh idle-archive grace period", async () => { + pipeline = createPipeline(buildDeps(db!)); + core = createMemoryCore( + pipeline, + resolveHome("openclaw", "/tmp/memos-mc-test"), + "test", + ); + await core.init(); + + seedCoreSkill("skill-reactivate", "reactivated skill"); + db!.repos.skills.setStatus("skill-reactivate" as SkillId, "archived", 1); + const before = Date.now(); + + await expect(core.reactivateSkill("skill-reactivate" as SkillId)).resolves.toMatchObject({ + status: "active", + }); + expect(db!.repos.skills.getById("skill-reactivate" as SkillId)?.lastUsedAt).toBeGreaterThanOrEqual( + before, + ); + }); + + it("archives idle low-eta skills while the pipeline stays running", async () => { + seedCoreSkill("skill-idle-running", "idle running skill"); + const stale = db!.repos.skills.getById("skill-idle-running" as SkillId)!; + db!.repos.skills.upsert({ + ...stale, + eta: 0.05, + createdAt: 1 as SkillRow["createdAt"], + updatedAt: 1 as SkillRow["updatedAt"], + lastUsedAt: null, + }); + + pipeline = createPipeline(buildDeps(db!)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(db!.repos.skills.getById("skill-idle-running" as SkillId)?.status).toBe("archived"); + }); }); describe("bootstrapMemoryCore", () => { diff --git a/apps/memos-local-plugin/tests/unit/skill/lifecycle-worker.test.ts b/apps/memos-local-plugin/tests/unit/skill/lifecycle-worker.test.ts new file mode 100644 index 000000000..08f6fd26b --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/skill/lifecycle-worker.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it, vi } from "vitest"; + +import { createSkillLifecycleWorker } from "../../../core/skill/lifecycle-worker.js"; +import { rootLogger } from "../../../core/logger/index.js"; + +describe("skill/lifecycle-worker", () => { + it("runs immediately, stays single-flight, and continues on its interval", async () => { + vi.useFakeTimers(); + try { + let releaseFirst!: () => void; + const firstRun = new Promise((resolve) => { + releaseFirst = resolve; + }); + const runLifecycle = vi + .fn<() => Promise>() + .mockReturnValueOnce(firstRun) + .mockResolvedValue(undefined); + const worker = createSkillLifecycleWorker({ + runLifecycle, + log: rootLogger.child({ channel: "test.skill.lifecycle-worker" }), + intervalMs: 1_000, + }); + + worker.start(); + await vi.advanceTimersByTimeAsync(2_000); + expect(runLifecycle).toHaveBeenCalledTimes(1); + + releaseFirst(); + await worker.flush(); + await vi.advanceTimersByTimeAsync(1_000); + expect(runLifecycle).toHaveBeenCalledTimes(2); + worker.stop(); + } finally { + vi.useRealTimers(); + } + }); + + it("logs scheduled failures and retries on the next interval", async () => { + vi.useFakeTimers(); + try { + const log = rootLogger.child({ channel: "test.skill.lifecycle-worker" }); + const warn = vi.spyOn(log, "warn").mockImplementation(() => undefined); + const runLifecycle = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error("scan failed")) + .mockResolvedValue(undefined); + const worker = createSkillLifecycleWorker({ + runLifecycle, + log, + intervalMs: 1_000, + }); + + worker.start(); + await vi.advanceTimersByTimeAsync(0); + expect(warn).toHaveBeenCalledWith("skill.lifecycle_worker.failed", { + err: "scan failed", + }); + + await vi.advanceTimersByTimeAsync(1_000); + expect(runLifecycle).toHaveBeenCalledTimes(2); + worker.stop(); + warn.mockRestore(); + } finally { + vi.useRealTimers(); + } + }); + + it("stops scheduled runs while allowing an explicit final run", async () => { + vi.useFakeTimers(); + try { + const runLifecycle = vi.fn<() => Promise>().mockResolvedValue(undefined); + const worker = createSkillLifecycleWorker({ + runLifecycle, + log: rootLogger.child({ channel: "test.skill.lifecycle-worker" }), + intervalMs: 1_000, + }); + + worker.start(); + await vi.advanceTimersByTimeAsync(0); + worker.stop(); + await vi.advanceTimersByTimeAsync(5_000); + expect(runLifecycle).toHaveBeenCalledTimes(1); + + await worker.runNow(); + expect(runLifecycle).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts b/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts index 666d7f4b6..a8c35468a 100644 --- a/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/subscriber.test.ts @@ -219,67 +219,6 @@ describe("skill/subscriber", () => { sub.dispose(); }); - it("continues after a full batch is archived concurrently", async () => { - handle = makeTmpDb(); - const h = handle; - for (let i = 0; i < 501; i++) { - seedSkill(h, { - id: `sk_concurrent_${i}` as never, - name: `concurrent_skill_${i}`, - status: "active", - eta: 0.05, - createdAt: 1 as never, - updatedAt: (i + 1) as never, - lastUsedAt: (i + 1) as never, - }); - } - - const bus = createSkillEventBus(); - const events: string[] = []; - bus.on("skill.status.changed", (event) => { - if (event.kind === "skill.status.changed") events.push(event.skillId); - }); - const log = rootLogger.child({ channel: "core.skill.subscriber" }); - const warnSpy = vi.spyOn(log, "warn").mockImplementation(() => undefined); - const archiveIdleBatch = h.repos.skills.archiveIdleBatch.bind( - h.repos.skills, - ); - const archiveSpy = vi - .spyOn(h.repos.skills, "archiveIdleBatch") - .mockImplementation((ids, input) => { - for (const id of ids) { - const index = Number(String(id).slice(String(id).lastIndexOf("_") + 1)); - if (index < 500) { - h.repos.skills.setStatus(id, "archived", input.updatedAt); - } - } - return archiveIdleBatch(ids, input); - }); - const sub = attachSkillSubscriber({ - l2Bus: createL2EventBus(), - rewardBus: createRewardEventBus(), - bus, - repos: h.repos, - embedder: null, - llm: null, - log, - config: makeSkillConfig({ minEtaForRetrieval: 0.1, idleArchiveMs: 1_000 }), - }); - - await sub.lifecycleTick(); - - expect(h.repos.skills.count({ status: "archived" })).toBe(501); - expect(events).toEqual(["sk_concurrent_500"]); - expect(warnSpy).toHaveBeenCalledWith("skill.idle_archive_stalled", { - candidateCount: 500, - cutoff: expect.any(Number), - minEtaForRetrieval: 0.1, - }); - sub.dispose(); - archiveSpy.mockRestore(); - warnSpy.mockRestore(); - }); - it("drains more than one 500-skill idle archive batch in one lifecycle tick", async () => { handle = makeTmpDb(); const h = handle; diff --git a/apps/memos-local-plugin/tests/unit/storage/repos.test.ts b/apps/memos-local-plugin/tests/unit/storage/repos.test.ts index ed1aa53db..4da3d77b8 100644 --- a/apps/memos-local-plugin/tests/unit/storage/repos.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/repos.test.ts @@ -351,41 +351,26 @@ describe("storage/repos — happy paths", () => { insertSkill("retrievable", "active", 0.1, 1, 100); insertSkill("already_archived", "archived", 0.05, 1, 100); - const candidates = repos.skills.listIdleArchiveCandidates({ + const archived = repos.skills.archiveNextIdleBatch({ minEtaForRetrieval: 0.1, cutoff: 9_000, + updatedAt: 10_000, limit: 500, }); - expect(candidates.map((skill) => skill.id)).toEqual(["never_used", "old_used"]); - expect(repos.skills.listIdleArchiveCandidates({ - minEtaForRetrieval: 0.1, - cutoff: 9_000, - limit: 1, - }).map((skill) => skill.id)).toEqual(["never_used"]); + expect(archived.map((skill) => skill.id).sort()).toEqual(["never_used", "old_used"]); + expect(repos.skills.getById("never_used")?.status).toBe("archived"); + expect(repos.skills.getById("old_used")?.status).toBe("archived"); + repos.skills.setStatus("old_used" as never, "active", 10_100); expect(repos.skills.recordUse("old_used", 9_500)).toBe(true); expect(repos.skills.getById("old_used")?.lastUsedAt).toBe(9_500); - expect( - repos.skills.archiveIdleBatch(["old_used"], { - minEtaForRetrieval: 0.1, - cutoff: 9_000, - updatedAt: 10_000, - }), - ).toEqual([]); - expect(repos.skills.getById("old_used")?.status).toBe("active"); - expect(repos.skills.listIdleArchiveCandidates({ + expect(repos.skills.archiveNextIdleBatch({ minEtaForRetrieval: 0.1, cutoff: 9_000, + updatedAt: 10_200, limit: 500, - }).map((skill) => skill.id)).toEqual(["never_used"]); - expect( - repos.skills.archiveIdleBatch(["never_used"], { - minEtaForRetrieval: 0.1, - cutoff: 9_000, - updatedAt: 10_000, - }), - ).toEqual(["never_used"]); - expect(repos.skills.getById("never_used")?.status).toBe("archived"); + })).toEqual([]); + expect(repos.skills.getById("old_used")?.status).toBe("active"); insertSkill("rollback_first", "active", 0.05, 1, 100); insertSkill("rollback_fail", "active", 0.05, 1, 100); @@ -397,14 +382,12 @@ describe("storage/repos — happy paths", () => { SELECT RAISE(ABORT, 'forced archive failure'); END`); expect(() => - repos.skills.archiveIdleBatch( - ["rollback_first", "rollback_fail"], - { - minEtaForRetrieval: 0.1, - cutoff: 9_000, - updatedAt: 10_000, - }, - ), + repos.skills.archiveNextIdleBatch({ + minEtaForRetrieval: 0.1, + cutoff: 9_000, + updatedAt: 10_300, + limit: 500, + }), ).toThrow(/forced archive failure/); expect(repos.skills.getById("rollback_first")?.status).toBe("active"); expect(repos.skills.getById("rollback_fail")?.status).toBe("active"); From c6aaf9d377d4ec3e3dd639e78a2902b0b01ec2f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Tue, 25 Aug 2026 11:46:01 +0800 Subject: [PATCH 24/34] fix(plugin): use OS home for bridge PID fallback Co-authored-by: asorry75 --- apps/memos-local-plugin/bridge.cts | 3 +- apps/memos-local-plugin/bridge.mts | 3 +- .../tests/unit/bridge/pid-file-path.test.ts | 29 +++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 apps/memos-local-plugin/tests/unit/bridge/pid-file-path.test.ts diff --git a/apps/memos-local-plugin/bridge.cts b/apps/memos-local-plugin/bridge.cts index 132af85f7..7e0f1820f 100644 --- a/apps/memos-local-plugin/bridge.cts +++ b/apps/memos-local-plugin/bridge.cts @@ -29,6 +29,7 @@ const path = require("node:path") as typeof import("node:path"); // eslint-disable-next-line @typescript-eslint/no-require-imports const fs = require("node:fs") as typeof import("node:fs"); // eslint-disable-next-line @typescript-eslint/no-require-imports +const { homedir } = require("node:os") as typeof import("node:os"); // eslint-disable-next-line @typescript-eslint/no-require-imports const url = require("node:url") as typeof import("node:url"); @@ -104,7 +105,7 @@ function pidFilePath( if (configuredHome) return path.join(path.resolve(configuredHome), "daemon", filename); const agentHome = agent === "hermes" ? ".hermes" : ".openclaw"; return path.join( - process.env.HOME ?? "/tmp", + homedir(), agentHome, "memos-plugin", "daemon", diff --git a/apps/memos-local-plugin/bridge.mts b/apps/memos-local-plugin/bridge.mts index 718172af2..4afca25f8 100644 --- a/apps/memos-local-plugin/bridge.mts +++ b/apps/memos-local-plugin/bridge.mts @@ -30,6 +30,7 @@ */ import * as childProcess from "node:child_process"; import * as fs from "node:fs"; +import { homedir } from "node:os"; import * as path from "node:path"; import { fileURLToPath } from "node:url"; @@ -87,7 +88,7 @@ function pidFilePath(agent: string, explicitHome?: string): string { if (configuredHome) return path.join(path.resolve(configuredHome), "daemon", PID_FILENAME); const agentHome = agent === "hermes" ? ".hermes" : ".openclaw"; return path.join( - process.env.HOME ?? "/tmp", + homedir(), agentHome, "memos-plugin", "daemon", diff --git a/apps/memos-local-plugin/tests/unit/bridge/pid-file-path.test.ts b/apps/memos-local-plugin/tests/unit/bridge/pid-file-path.test.ts new file mode 100644 index 000000000..cf7f51dfa --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/bridge/pid-file-path.test.ts @@ -0,0 +1,29 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +import { describe, expect, it } from "vitest"; + +describe("bridge PID file path", () => { + for (const entry of ["bridge.cts", "bridge.mts"]) { + it(`${entry} preserves configured homes and falls back to the OS home`, () => { + const source = readFileSync(resolve(entry), "utf8"); + const start = source.indexOf("function pidFilePath"); + const end = source.indexOf("function readPidFile", start); + + expect(start, `${entry}: pidFilePath() not found`).toBeGreaterThanOrEqual(0); + expect(end, `${entry}: readPidFile() not found`).toBeGreaterThan(start); + + const pidFilePathSource = source.slice(start, end); + const configuredHomeGuard = pidFilePathSource.indexOf("if (configuredHome)"); + const osHomeFallback = pidFilePathSource.indexOf("homedir()"); + + expect(configuredHomeGuard, `${entry}: configured home guard missing`).toBeGreaterThanOrEqual( + 0, + ); + expect(osHomeFallback, `${entry}: OS home fallback missing`).toBeGreaterThan( + configuredHomeGuard, + ); + expect(pidFilePathSource).not.toMatch(/process\.env\.HOME|["']\/tmp["']/); + }); + } +}); From e1530908e1e3db19932ec5dcdff5ab8deb13c2fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Tue, 25 Aug 2026 17:41:56 +0800 Subject: [PATCH 25/34] fix(plugin): scope secret environment fallbacks --- apps/memos-local-plugin/core/config/index.ts | 71 ++++++----- .../unit/config/resolve-secret-env.test.ts | 112 ++++++++++++++++-- 2 files changed, 147 insertions(+), 36 deletions(-) diff --git a/apps/memos-local-plugin/core/config/index.ts b/apps/memos-local-plugin/core/config/index.ts index 70791d6a9..1b15efc0c 100644 --- a/apps/memos-local-plugin/core/config/index.ts +++ b/apps/memos-local-plugin/core/config/index.ts @@ -103,17 +103,15 @@ export function resolveConfig(raw: unknown, warnings?: string[], agent?: string) // 2. Value is the mask sentinel `__memos_secret__` or empty string // -> derive the env var from the field path itself // (llm.apiKey -> LLM_API_KEY, hub.teamToken -> HUB_TEAM_TOKEN, - // skillEvolver.apiKey -> SKILL_EVOLVER_API_KEY, …). The generic - // OPENCODE_GO/ZEN fallback is applied ONLY to the primary - // `llm.apiKey`; per-component overrides (l3Llm, skillEvolver) - // and non-LLM secrets (embedding, hub tokens) never borrow an - // unrelated provider's key — that would cause cross-provider - // auth failures or unexpected billing on the wrong account. + // skillEvolver.apiKey -> SKILL_EVOLVER_API_KEY, …). Provider-specific + // OpenCode fallbacks only apply when the primary LLM points at the + // matching opencode.ai endpoint; unrelated providers and dedicated + // config slots never borrow those keys. // 3. Otherwise leave the value untouched. // - // Any secret leaf that references an env var that is not set emits a - // warning so the operator gets an actionable log message instead of - // silent auth failures on the next LLM call. + // A masked sentinel or explicit env reference with no backing variable + // emits a warning. A plain empty string does not: empty keys are valid for + // local/host providers and disabled optional integrations. // // The mask itself is never used as a credential, and the on-disk write // stays masked (security preserved); this is read-side only. @@ -179,7 +177,7 @@ function resolveSecretEnv(cleaned: Record, warnings?: string[]) if (typeof val !== "string") continue; let envName: string | null = null; - let genericFallbacks = false; + let warnIfMissing = false; if (val.startsWith("${") && val.endsWith("}")) { const name = val.slice(2, -1); if (!ENV_REF_ALLOWLIST.test(name)) { @@ -190,6 +188,7 @@ function resolveSecretEnv(cleaned: Record, warnings?: string[]) continue; } envName = name; + warnIfMissing = true; } else if (val === "__memos_secret__" || val === "") { // Derive env var name from the field path itself so every entry // in SECRET_FIELD_PATHS is resolvable, not just the ones whose @@ -202,29 +201,14 @@ function resolveSecretEnv(cleaned: Record, warnings?: string[]) // hub.userToken → HUB_USER_TOKEN const parent = keys[keys.length - 2] ?? ""; envName = `${camelToUpperSnake(parent)}_${camelToUpperSnake(leaf)}`; - // OPENCODE_GO_API_KEY / OPENCODE_ZEN_API_KEY are only meaningful - // for the primary `llm.apiKey`. Per-component overrides - // (l3Llm.apiKey, skillEvolver.apiKey) and non-LLM secrets - // (embedding.apiKey, hub.*Token) must not silently borrow an - // unrelated provider's key — doing so causes cross-provider auth - // failures and unexpected billing on the wrong account when the - // component is configured for a different provider entirely. - genericFallbacks = parent === "llm" && leaf === "apiKey"; + warnIfMissing = val === "__memos_secret__"; } if (!envName) continue; - const envVal = - process.env[envName] ?? - (genericFallbacks - ? (process.env.OPENCODE_GO_API_KEY ?? process.env.OPENCODE_ZEN_API_KEY) - : undefined); + const envVal = process.env[envName] ?? resolveOpenCodeApiKey(cleaned, dotted); if (envVal) { (cursor as Record)[leaf] = envVal; - } else { - // Explicit-reference case: the user asked for env expansion but - // the target is unset. Mask/empty case: we walked the path-based - // convention and nothing was set. Both perpetuate the original - // bug (silent auth failure on next LLM call) unless we log it. + } else if (warnIfMissing) { warnings?.push( `config: '${dotted}' references env var '${envName}' but it is not set — ` + `field left as placeholder and auth will fail on the next call` @@ -233,6 +217,37 @@ function resolveSecretEnv(cleaned: Record, warnings?: string[]) } } +/** + * Resolve the provider-specific OpenCode fallback for the primary LLM only. + * The hostname, provider and endpoint tier must all match so an OpenCode key + * can never be sent to Anthropic, Gemini or another OpenAI-compatible host. + */ +function resolveOpenCodeApiKey( + cleaned: Record, + dotted: string, +): string | undefined { + if (dotted !== "llm.apiKey" || !isPlainObject(cleaned.llm)) return undefined; + if (cleaned.llm.provider !== "openai_compatible") return undefined; + + const endpoint = cleaned.llm.endpoint; + if (typeof endpoint !== "string" || endpoint.length === 0) return undefined; + + try { + const url = new URL(endpoint); + if (url.hostname !== "opencode.ai") return undefined; + if (/^\/zen\/go(?:\/|$)/.test(url.pathname)) { + return process.env.OPENCODE_GO_API_KEY; + } + if (/^\/zen(?:\/|$)/.test(url.pathname)) { + return process.env.OPENCODE_ZEN_API_KEY; + } + } catch { + // Schema validation reports malformed endpoints later. Secret resolution + // must not broaden fallback scope just because parsing failed here. + } + return undefined; +} + /** * camelCase → UPPER_SNAKE_CASE for deriving env var names from config * field paths. Only inserts an underscore at a lowercase/digit → uppercase diff --git a/apps/memos-local-plugin/tests/unit/config/resolve-secret-env.test.ts b/apps/memos-local-plugin/tests/unit/config/resolve-secret-env.test.ts index 9054010a2..233951231 100644 --- a/apps/memos-local-plugin/tests/unit/config/resolve-secret-env.test.ts +++ b/apps/memos-local-plugin/tests/unit/config/resolve-secret-env.test.ts @@ -1,7 +1,8 @@ import { afterEach, describe, expect, it } from "vitest"; -import { resolveConfig } from "../../../core/config/index.js"; +import { loadConfig, resolveConfig } from "../../../core/config/index.js"; import { SECRET_FIELD_PATHS } from "../../../core/config/defaults.js"; +import { makeTmpHome } from "../../helpers/tmp-home.js"; const ORIGINAL_ENV = { ...process.env }; @@ -18,21 +19,116 @@ describe("resolveConfig secret env fallback", () => { it("resolves the __memos_secret__ mask sentinel from env", () => { process.env.OPENCODE_GO_API_KEY = "sk-mask-resolved"; - const cfg = resolveConfig({ llm: { apiKey: "__memos_secret__" } }); + const cfg = resolveConfig({ + llm: { + provider: "openai_compatible", + endpoint: "https://opencode.ai/zen/go/v1", + apiKey: "__memos_secret__", + }, + }); expect(cfg.llm.apiKey).toBe("sk-mask-resolved"); }); it("resolves empty string secret fields from env", () => { process.env.OPENCODE_ZEN_API_KEY = "sk-empty-resolved"; - const cfg = resolveConfig({ llm: { apiKey: "" } }); + const cfg = resolveConfig({ + llm: { + provider: "openai_compatible", + endpoint: "https://opencode.ai/zen/v1", + apiKey: "", + }, + }); expect(cfg.llm.apiKey).toBe("sk-empty-resolved"); }); + it("does not send an OpenCode key to a different LLM provider", () => { + delete process.env.LLM_API_KEY; + process.env.OPENCODE_GO_API_KEY = "sk-opencode-only"; + process.env.OPENCODE_ZEN_API_KEY = "sk-opencode-zen-only"; + + const cfg = resolveConfig({ + llm: { + provider: "anthropic", + endpoint: "https://api.anthropic.com", + apiKey: "__memos_secret__", + }, + }); + + expect(cfg.llm.apiKey).toBe("__memos_secret__"); + }); + + it("keeps OpenCode Go and Zen endpoint keys isolated", () => { + delete process.env.LLM_API_KEY; + process.env.OPENCODE_GO_API_KEY = "sk-go-only"; + delete process.env.OPENCODE_ZEN_API_KEY; + + const zenConfig = resolveConfig({ + llm: { + provider: "openai_compatible", + endpoint: "https://opencode.ai/zen/v1", + apiKey: "__memos_secret__", + }, + }); + expect(zenConfig.llm.apiKey).toBe("__memos_secret__"); + + delete process.env.OPENCODE_GO_API_KEY; + process.env.OPENCODE_ZEN_API_KEY = "sk-zen-only"; + const goConfig = resolveConfig({ + llm: { + provider: "openai_compatible", + endpoint: "https://opencode.ai/zen/go/v1", + apiKey: "__memos_secret__", + }, + }); + expect(goConfig.llm.apiKey).toBe("__memos_secret__"); + }); + + it("does not warn for intentionally empty optional secrets", () => { + delete process.env.EMBEDDING_API_KEY; + delete process.env.LLM_API_KEY; + delete process.env.OPENCODE_GO_API_KEY; + delete process.env.OPENCODE_ZEN_API_KEY; + delete process.env.HUB_TEAM_TOKEN; + const warnings: string[] = []; + + resolveConfig( + { + embedding: { provider: "local", apiKey: "" }, + llm: { provider: "host", apiKey: "" }, + hub: { enabled: false, teamToken: "" }, + }, + warnings, + ); + + expect(warnings).toEqual([]); + }); + + it("restores a masked disk secret when config is loaded again", async () => { + process.env.LLM_API_KEY = "sk-restart-restored"; + const ctx = await makeTmpHome({ + agent: "hermes", + configYaml: [ + "version: 1", + "llm:", + " provider: openai_compatible", + " endpoint: https://example.com/v1", + " apiKey: __memos_secret__", + ].join("\n"), + }); + + try { + const restarted = await loadConfig(ctx.home, "hermes"); + expect(restarted.fromDisk).toBe(true); + expect(restarted.config.llm.apiKey).toBe("sk-restart-restored"); + } finally { + await ctx.cleanup(); + } + }); + it("uses per-path env conventions — every secret path resolves from its own env var", () => { - // OPENCODE_GO_API_KEY is only the generic fallback for the *primary* - // llm.apiKey — l3Llm / skillEvolver / hub / embedding all get their - // own path-derived env var and never silently borrow the LLM key. - process.env.OPENCODE_GO_API_KEY = "sk-llm"; + // Every path has its own environment variable and never silently borrows + // a provider-specific key intended for another config slot. + process.env.LLM_API_KEY = "sk-llm"; process.env.EMBEDDING_API_KEY = "sk-embed"; process.env.L3_LLM_API_KEY = "sk-l3"; process.env.SKILL_EVOLVER_API_KEY = "sk-skill"; @@ -159,7 +255,7 @@ describe("resolveConfig secret env fallback", () => { }); it("never mutates the caller's raw config object", () => { - process.env.OPENCODE_GO_API_KEY = "sk-llm"; + process.env.LLM_API_KEY = "sk-llm"; const raw = { llm: { apiKey: "__memos_secret__" } }; const cfg = resolveConfig(raw); expect(cfg.llm.apiKey).toBe("sk-llm"); From 602f60ff59a3d9cafcf02eb7fa4c93fd1552c390 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Tue, 25 Aug 2026 20:22:13 +0800 Subject: [PATCH 26/34] fix(plugin): make free-form config paths explicit --- .../core/config/defaults.ts | 13 ++++++++++++ apps/memos-local-plugin/core/config/index.ts | 15 ++++++++----- .../config/llm-max-tokens-headers.test.ts | 21 +++++++++++++++++++ .../pipeline/bootstrap-llm-config.test.ts | 10 +++++++++ 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/apps/memos-local-plugin/core/config/defaults.ts b/apps/memos-local-plugin/core/config/defaults.ts index 45685ded5..2575a8405 100644 --- a/apps/memos-local-plugin/core/config/defaults.ts +++ b/apps/memos-local-plugin/core/config/defaults.ts @@ -362,6 +362,19 @@ export const DEFAULT_CONFIG: ResolvedConfig = { }, }; +/** + * Object-valued config slots whose child keys are user-defined rather than + * fields in `DEFAULT_CONFIG`. Keep this list explicit: treating every empty + * default object as a free-form map would silently disable unknown-key + * warnings for any future structured config section that starts out empty. + */ +export const FREE_FORM_CONFIG_PATHS: readonly string[] = Object.freeze([ + "llm.headers", + "l3Llm.headers", + "skillEvolver.headers", + "logging.channels", +]); + /** * Set of dotted-path field names whose values must never be sent to the * viewer or any non-localhost surface. Used by `server/routes/config.ts`. diff --git a/apps/memos-local-plugin/core/config/index.ts b/apps/memos-local-plugin/core/config/index.ts index 6ac421274..ae3991017 100644 --- a/apps/memos-local-plugin/core/config/index.ts +++ b/apps/memos-local-plugin/core/config/index.ts @@ -18,7 +18,12 @@ import { MemosError } from "../../agent-contract/errors.js"; import type { ResolvedHome } from "./paths.js"; import { resolveHome } from "./paths.js"; import { ConfigSchema, type ResolvedConfig } from "./schema.js"; -import { DEFAULT_CONFIG, SECRET_FIELD_PATHS, effectiveViewerPort } from "./defaults.js"; +import { + DEFAULT_CONFIG, + FREE_FORM_CONFIG_PATHS, + SECRET_FIELD_PATHS, + effectiveViewerPort, +} from "./defaults.js"; import { migrateHermesViewerPort } from "./migrations.js"; import { parseYaml } from "./yaml.js"; @@ -323,10 +328,10 @@ function pruneUnknown( continue; } if (isPlainObject(v) && isPlainObject((defaults as Record)[k])) { - if (Object.keys((defaults as Record)[k] as Record).length === 0) { - // Empty-object default slot = free-form map (e.g. llm.headers, a - // Record). Keep the whole user object as-is; recursing - // would warn on every user key. + if (FREE_FORM_CONFIG_PATHS.includes(path)) { + // Explicitly declared free-form maps keep user-defined child keys. + // Other empty default objects remain structured config sections and + // continue to report unknown nested keys. out[k] = v; continue; } diff --git a/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts b/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts index 19f424856..21dd9de88 100644 --- a/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts +++ b/apps/memos-local-plugin/tests/unit/config/llm-max-tokens-headers.test.ts @@ -1,8 +1,29 @@ import { describe, expect, it } from "vitest"; import { DEFAULT_CONFIG, resolveConfig } from "../../../core/config/index.js"; +import { FREE_FORM_CONFIG_PATHS } from "../../../core/config/defaults.js"; describe("resolveConfig llm.maxTokens + llm.headers", () => { + it("uses an explicit allowlist for free-form config maps", () => { + expect(FREE_FORM_CONFIG_PATHS).toEqual([ + "llm.headers", + "l3Llm.headers", + "skillEvolver.headers", + "logging.channels", + ]); + expect(Object.isFrozen(FREE_FORM_CONFIG_PATHS)).toBe(true); + + const warnings: string[] = []; + const cfg = resolveConfig( + { + logging: { channels: { "core.l2.cross-task": "debug" } }, + }, + warnings, + ); + expect(cfg.logging.channels).toEqual({ "core.l2.cross-task": "debug" }); + expect(warnings).toEqual([]); + }); + it("accepts llm.maxTokens and llm.headers without unknown-key warnings", () => { const warnings: string[] = []; const cfg = resolveConfig( diff --git a/apps/memos-local-plugin/tests/unit/pipeline/bootstrap-llm-config.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/bootstrap-llm-config.test.ts index 512c4cdde..fd62c903d 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/bootstrap-llm-config.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/bootstrap-llm-config.test.ts @@ -92,6 +92,9 @@ skillEvolver: openRouter: true model: skill-model apiKey: sk-test + maxTokens: 6144 + headers: + X-Evolver: skill-header providerIgnore: - together providerOrder: @@ -102,6 +105,9 @@ l3Llm: openRouter: true model: l3-model apiKey: sk-test + maxTokens: 8192 + headers: + X-L3: l3-header providerIgnore: - novita providerOrder: @@ -123,12 +129,16 @@ l3Llm: providerIgnore: ["together"], providerOrder: ["anthropic"], openRouter: true, + maxTokens: 6_144, + headers: { "X-Evolver": "skill-header" }, }); expect(capturedLlmConfigs.find((cfg) => cfg.model === "l3-model")).toMatchObject({ providerIgnore: ["novita"], providerOrder: ["openai"], openRouter: true, reasoning: { enabled: true, maxTokens: 4_000 }, + maxTokens: 8_192, + headers: { "X-L3": "l3-header" }, }); }); From 4a095217f9b333b6890095044d9e8b87c17d9861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Wed, 26 Aug 2026 12:02:12 +0800 Subject: [PATCH 27/34] fix(plugin): harden startup recovery shutdown --- apps/memos-local-plugin/bridge.cts | 8 ++- apps/memos-local-plugin/bridge.mts | 26 ++++++--- .../core/pipeline/memory-core.ts | 41 ++++++++++++-- .../core/pipeline/orchestrator.ts | 19 +++++-- .../memos-local-plugin/core/pipeline/types.ts | 9 +++- .../unit/logger/signal-ownership.test.ts | 5 +- .../tests/unit/pipeline/memory-core.test.ts | 53 +++++++++++++++++++ .../tests/unit/startup-recovery.test.ts | 16 ++++++ 8 files changed, 159 insertions(+), 18 deletions(-) diff --git a/apps/memos-local-plugin/bridge.cts b/apps/memos-local-plugin/bridge.cts index 7e0f1820f..5e533779e 100644 --- a/apps/memos-local-plugin/bridge.cts +++ b/apps/memos-local-plugin/bridge.cts @@ -43,7 +43,13 @@ const BRIDGE_STATUS_FILE = "bridge-status.json"; const SHUTDOWN_TIMEOUT_MS = 20_000; function withShutdownTimeout(p: Promise): Promise { - return Promise.race([p, new Promise((r) => setTimeout(r, SHUTDOWN_TIMEOUT_MS))]); + let timer: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timer = setTimeout(resolve, SHUTDOWN_TIMEOUT_MS); + }); + return Promise.race([p, timeout]).finally(() => { + if (timer) clearTimeout(timer); + }); } interface BridgeArgs { diff --git a/apps/memos-local-plugin/bridge.mts b/apps/memos-local-plugin/bridge.mts index 4afca25f8..d271fc7f2 100644 --- a/apps/memos-local-plugin/bridge.mts +++ b/apps/memos-local-plugin/bridge.mts @@ -40,6 +40,20 @@ const __dirname = path.dirname(__filename); const BRIDGE_STATUS_HEARTBEAT_MS = 5_000; const BRIDGE_STATUS_STALE_MS = 20_000; const BRIDGE_STATUS_FILE = "bridge-status.json"; +// Keep both executable bridge entries within the same process-level budget. +// Core shutdown has its own cooperative recovery cancellation, while this +// outer deadline guarantees a broken provider cannot orphan the bridge. +const SHUTDOWN_TIMEOUT_MS = 20_000; + +function withShutdownTimeout(p: Promise): Promise { + let timer: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timer = setTimeout(resolve, SHUTDOWN_TIMEOUT_MS); + }); + return Promise.race([p, timeout]).finally(() => { + if (timer) clearTimeout(timer); + }); +} interface BridgeArgs { daemon: boolean; @@ -400,13 +414,13 @@ async function main(): Promise { process.stderr.write( `bridge: daemon port :${viewerPort} still in use after ${maxBindAttempts}s — exiting.\n`, ); - await core.shutdown(); + await withShutdownTimeout(core.shutdown()); process.exit(1); } process.stderr.write( `bridge: daemon viewer failed: ${(err as Error)?.message ?? String(err)}\n`, ); - await core.shutdown(); + await withShutdownTimeout(core.shutdown()); process.exit(1); } } @@ -416,7 +430,7 @@ async function main(): Promise { removeOwnedPidFile(); try { await viewer!.close(); } catch { /* best-effort */ } try { - await core.shutdown(); + await withShutdownTimeout(core.shutdown()); } catch { // clear-data already shuts the core down before removing SQLite. // The signal still has to terminate the daemon so the supervisor @@ -493,7 +507,7 @@ async function main(): Promise { /* best-effort */ } } - await waitForShutdown(core, activeStdio); + await withShutdownTimeout(waitForShutdown(core, activeStdio)); process.exit(0); }; @@ -514,7 +528,7 @@ async function main(): Promise { if (viewer!.closed) { clearInterval(keepalive); removeOwnedPidFile(); - void core.shutdown().then(() => process.exit(0)); + void withShutdownTimeout(core.shutdown()).then(() => process.exit(0)); } }, 5_000); (keepalive as unknown as { unref?: () => void }).unref?.(); @@ -523,7 +537,7 @@ async function main(): Promise { // No viewer (headless bridge) — clean exit. removeOwnedPidFile(); - await core.shutdown(); + await withShutdownTimeout(core.shutdown()); process.exit(0); } diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index 9b0fdbedc..bd64d5e08 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -577,6 +577,8 @@ export interface CreateMemoryCoreOptions { onShutdown?: () => void | Promise; /** Optional telemetry instance for ARMS RUM reporting. */ telemetry?: import("../telemetry/index.js").Telemetry | null; + /** Startup-recovery grace used by shutdown. Defaults to 15 seconds. */ + startupRecoveryShutdownGraceMs?: number; } /** @@ -596,6 +598,10 @@ export function createMemoryCore( const bootAt = Date.now(); const log = rootLogger.child({ channel: "core.pipeline.memory-core" }); const autoRecoveryEnabled = options.autoRecovery ?? true; + const startupRecoveryShutdownGraceMs = Math.max( + 0, + options.startupRecoveryShutdownGraceMs ?? 15_000, + ); let telemetry = options.telemetry ?? null; let initialized = false; let shutDown = false; @@ -718,6 +724,7 @@ export function createMemoryCore( // detach the slow recovery to this promise. `waitForStartupRecovery` // exposes it so tests can opt back into the deterministic semantics. let startupRecoveryPromise: Promise = Promise.resolve(); + let startupRecoveryCancelled = false; let lastStaleScan = 0; let lastDirtyClosedScan = 0; async function autoFinalizeStaleTasks(): Promise { @@ -1517,6 +1524,7 @@ export function createMemoryCore( async function recoverOpenEpisodesAsSessionEnd( orphans: Array }>, ): Promise { + if (startupRecoveryCancelled) return; const endedAt = Date.now(); log.info("init.orphan_episodes.session_end_recover", { count: orphans.length }); debugStartupRecovery("H1", "startup_recovery_scan", { @@ -1547,6 +1555,7 @@ export function createMemoryCore( }); try { for (const ep of orphans) { + if (startupRecoveryCancelled) break; if (isLightweightEpisode(ep)) continue; try { const episodeId = ep.id as EpisodeId; @@ -1600,7 +1609,9 @@ export function createMemoryCore( try { await handle.flush(); + if (startupRecoveryCancelled) return; for (const episodeId of needsRewardFallback) { + if (startupRecoveryCancelled) break; if (captureFailedInBatch.has(episodeId)) { log.warn("init.orphan_recovery.reward_fallback_skipped", { episodeId, @@ -1617,6 +1628,7 @@ export function createMemoryCore( }); } } + if (startupRecoveryCancelled) return; await handle.flush(); debugStartupRecovery("H5", "startup_recovery_flush_done", { recoveredCount: orphans.length, @@ -1649,11 +1661,13 @@ export function createMemoryCore( async function recoverDirtyClosedEpisodes( episodes: Array }>, ): Promise { + if (startupRecoveryCancelled) return; log.info("init.dirty_closed_episodes.rescore", { count: episodes.length }); // Snapshot the prior failure counters so we can increment them later // (after the bus chain settles) without an extra DB read. const priorFailedAttempts = new Map(); for (const ep of episodes) { + if (startupRecoveryCancelled) break; if (isLightweightEpisode(ep)) continue; const episodeId = ep.id as EpisodeId; const endedAt = ep.endedAt ?? Date.now(); @@ -1680,6 +1694,7 @@ export function createMemoryCore( }); } await handle.flush(); + if (startupRecoveryCancelled) return; // After the reward / reflect chain has finished, account for the // outcome: clear `meta.rewardDirty` on episodes that are no longer // dirty (success), bump `failedAttempts + lastFailureAt` on episodes @@ -1966,6 +1981,7 @@ export function createMemoryCore( // wait, a fast `init → shutdown` race during tests or a quick // gateway reload would close SQLite while reflect / reward is // mid-flush, producing `SQLITE_MISUSE` noise on the way down. + let startupRecoveryTimedOut = false; try { // Bound the wait: a slow / flaky LLM during startup recovery of a // large dirty episode must not hold shutdown hostage until the @@ -1974,9 +1990,23 @@ export function createMemoryCore( // episodes carry rewardDirty.failedAttempts and the periodic // rescore re-runs them — so nothing is lost by proceeding after a // short grace. Fast init→shutdown races still get their grace. - await withTimeout(startupRecoveryPromise, 15_000, "startup_recovery_shutdown_timeout"); - } catch { - /* already logged inside the recovery promise */ + await withTimeout( + startupRecoveryPromise, + startupRecoveryShutdownGraceMs, + "startup_recovery_shutdown_timeout", + ); + } catch (err) { + if ( + err instanceof Error && + err.message === "startup_recovery_shutdown_timeout" + ) { + startupRecoveryTimedOut = true; + startupRecoveryCancelled = true; + log.warn("startup_recovery.shutdown_timeout", { + timeoutMs: startupRecoveryShutdownGraceMs, + action: "cancel_and_shutdown_pipeline", + }); + } } try { await hubRuntime?.stop(); @@ -1985,7 +2015,10 @@ export function createMemoryCore( err: err instanceof Error ? err.message : String(err), }); } - await handle.shutdown("memory-core.shutdown"); + await handle.shutdown( + "memory-core.shutdown", + startupRecoveryTimedOut ? { flushGraceMs: 0 } : undefined, + ); } finally { disposeTurnStartApiLogSessionListener(); turnStartApiLogBySession.clear(); diff --git a/apps/memos-local-plugin/core/pipeline/orchestrator.ts b/apps/memos-local-plugin/core/pipeline/orchestrator.ts index 97bdac299..17e444bd0 100644 --- a/apps/memos-local-plugin/core/pipeline/orchestrator.ts +++ b/apps/memos-local-plugin/core/pipeline/orchestrator.ts @@ -56,6 +56,7 @@ import type { PipelineBuses, PipelineDeps, PipelineHandle, + PipelineShutdownOptions, RecordToolOutcomeInput, TurnEndResult, } from "./types.js"; @@ -1651,7 +1652,10 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { await embeddingRetryWorker.flush(); } - async function shutdown(reason: string = "shutdown"): Promise { + async function shutdown( + reason: string = "shutdown", + options: PipelineShutdownOptions = {}, + ): Promise { log.info("pipeline.shutdown.begin", { reason }); // Stop admitting retry jobs, but preserve a bounded grace period for raw // capture and downstream enrichment. Hermes' bridge owns a 20s outer @@ -1659,15 +1663,20 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { // discarding every single-shot session's enrichment immediately. skillLifecycleWorker.stop(); embeddingRetryWorker.stop(); + const flushGraceMs = Math.max(0, options.flushGraceMs ?? 15_000); + const abortWaitMs = Math.max(0, options.abortWaitMs ?? 4_000); + if (flushGraceMs === 0) { + foregroundResources.shutdown(reason); + } const flushPromise = flush(); try { - const completed = await settlesWithin(flushPromise, 15_000); + const completed = await settlesWithin(flushPromise, flushGraceMs); if (!completed) { - log.warn("pipeline.flush_timeout", { reason, timeoutMs: 15_000 }); + log.warn("pipeline.flush_timeout", { reason, timeoutMs: flushGraceMs }); foregroundResources.shutdown(reason); - const aborted = await settlesWithin(flushPromise, 4_000); + const aborted = await settlesWithin(flushPromise, abortWaitMs); if (!aborted) { - log.warn("pipeline.flush_abandoned", { reason, abortWaitMs: 4_000 }); + log.warn("pipeline.flush_abandoned", { reason, abortWaitMs }); } } } catch (err) { diff --git a/apps/memos-local-plugin/core/pipeline/types.ts b/apps/memos-local-plugin/core/pipeline/types.ts index b37818344..4e20bfada 100644 --- a/apps/memos-local-plugin/core/pipeline/types.ts +++ b/apps/memos-local-plugin/core/pipeline/types.ts @@ -248,12 +248,19 @@ export interface PipelineHandle { // Imperative helpers. flush(): Promise; - shutdown(reason?: string): Promise; + shutdown(reason?: string, options?: PipelineShutdownOptions): Promise; /** Compose a retrieval-deps instance scoped to this pipeline. Used by tests. */ retrievalDeps(): RetrievalDeps; } +export interface PipelineShutdownOptions { + /** Grace before aborting provider calls and queued background work. */ + flushGraceMs?: number; + /** Final drain window after abort before subscribers are detached. */ + abortWaitMs?: number; +} + export interface PipelineBuses { session: SessionEventBus; capture: CaptureEventBus; diff --git a/apps/memos-local-plugin/tests/unit/logger/signal-ownership.test.ts b/apps/memos-local-plugin/tests/unit/logger/signal-ownership.test.ts index 9e34440c4..c611e6b24 100644 --- a/apps/memos-local-plugin/tests/unit/logger/signal-ownership.test.ts +++ b/apps/memos-local-plugin/tests/unit/logger/signal-ownership.test.ts @@ -17,7 +17,10 @@ describe("process signal ownership", () => { const source = readFileSync(resolve(entry), "utf8"); expect(source).toMatch(/process\.on\("SIGINT"/); expect(source).toMatch(/process\.on\("SIGTERM"/); - expect(source).toMatch(/await (?:withShutdownTimeout\()?core\.shutdown\(\)/); + expect(source).toContain("const SHUTDOWN_TIMEOUT_MS = 20_000"); + expect(source).toContain("function withShutdownTimeout(p: Promise): Promise"); + expect(source).not.toMatch(/(? { + db!.repos.sessions.upsert({ + id: "se_stalled_recovery", + agent: "openclaw", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt: 1_700_000_000_000, + lastSeenAt: 1_700_000_000_000, + meta: {}, + }); + db!.repos.episodes.insert({ + id: "ep_stalled_recovery", + sessionId: "se_stalled_recovery", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt: 1_700_000_000_000, + endedAt: null, + traceIds: [], + rTask: null, + status: "open", + meta: {}, + }); + + pipeline = createPipeline(buildDeps(db!)); + const originalShutdown = pipeline.shutdown.bind(pipeline); + pipeline.flush = vi.fn(() => new Promise(() => {})); + const shutdownSpy = vi.fn( + async ( + reason?: string, + options?: Parameters[1], + ) => originalShutdown(reason, { ...options, abortWaitMs: 0 }), + ); + pipeline.shutdown = shutdownSpy; + core = createMemoryCore( + pipeline, + resolveHome("openclaw", "/tmp/memos-mc-test"), + "issue2252-stalled-recovery", + { startupRecoveryShutdownGraceMs: 10 }, + ); + + await core.init(); + await expect(core.shutdown()).resolves.toBeUndefined(); + expect(shutdownSpy).toHaveBeenCalledWith( + "memory-core.shutdown", + { flushGraceMs: 0 }, + ); + + core = null; + pipeline = null; + }); + it("does not rescore a closed episode whose only mismatch is a ghost trace ID (#1966)", async () => { // Regression guard for https://github.com/MemTensor/MemOS/issues/1966. // A dangling ID in trace_ids_json must not make reward coverage look dirty diff --git a/apps/memos-local-plugin/tests/unit/startup-recovery.test.ts b/apps/memos-local-plugin/tests/unit/startup-recovery.test.ts index 94441a273..31df85ebf 100644 --- a/apps/memos-local-plugin/tests/unit/startup-recovery.test.ts +++ b/apps/memos-local-plugin/tests/unit/startup-recovery.test.ts @@ -15,6 +15,14 @@ function initBody(): string { return source.slice(start, end); } +function shutdownBody(): string { + const start = source.indexOf(" async function shutdown(): Promise {"); + expect(start, "shutdown() function should be present").toBeGreaterThanOrEqual(0); + const end = source.indexOf("\n async function health", start + 1); + expect(end, "shutdown() should be followed by health()").toBeGreaterThan(start); + return source.slice(start, end); +} + function stripBackgroundRecoveryCallback(body: string): string { return body.replace( /startupRecoveryPromise = \(async \(\) => \{[\s\S]*?\n\s*\}\)\(\);/g, @@ -32,4 +40,12 @@ describe("memory-core startup recovery", () => { expect(body).toContain("startupRecoveryPromise = (async () => {"); expect(body).not.toContain("await startupRecoveryPromise"); }); + + it("cancels and reports startup recovery that exceeds the shutdown grace", () => { + const body = shutdownBody(); + + expect(body).toContain('log.warn("startup_recovery.shutdown_timeout"'); + expect(body).toContain("startupRecoveryCancelled = true"); + expect(body).toContain("flushGraceMs: 0"); + }); }); From d22dbb87d62daf2cd8666d47d9f690d76fba00c9 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Wed, 26 Aug 2026 04:23:05 +0000 Subject: [PATCH 28/34] fix(storage): add idx_traces_ts to unblock event loop on newest-first trace reads Problem ------- The daemon froze solid on installs with a large `traces` table: HTTP requests were accepted by the kernel backlog but never answered, boot took 40-60s of near-100% CPU, and any liveness watchdog restart-looped the process forever (300+ restarts/day observed in production). Every doomed generation re-ran the scan at boot and was killed mid-scan, making the storm self-sustaining. Root cause (CPU-profiled, 28% of all samples in one statement) ------- `traces.list({limit:1})` -- used by `latestTraceTs()` (3x per `/api/v1/health` request) and by the pipeline's recent-events replay at bootstrap -- issues `SELECT ... FROM traces ORDER BY ts DESC, id DESC LIMIT 1`. No `traces` index leads with bare `ts` (`EXPLAIN QUERY PLAN`: `SCAN traces` + `USE TEMP B-TREE FOR ORDER BY`), so each call was a full table scan + sort. better-sqlite3 runs statements synchronously on the JS event loop, so the scan blocked ALL request handling while it ran. Fix --- - 013-traces-ts-index.sql: `CREATE INDEX IF NOT EXISTS idx_traces_ts ON traces(ts DESC, id DESC)` -- turns the lookup into an index seek. - migrator.ts: same tableExists guard as 012 for partial test schemas, plus a release-train heal: DBs migrated by the other train carry schema_migrations rows whose VERSION numbers collide under different NAMES (observed: 13 = 'skill-repair-origin', 14 = 'episode-outcome'), which silently skipped any same-numbered migration from this build. Additive, guarded migrations now still run under such collisions and repair their bookkeeping row via upsert; all others keep the conservative skip behaviour. - migrator.test.ts: regression tests for both behaviours (8/8 pass). Validation ---------- - EXPLAIN after: `SCAN traces USING INDEX idx_traces_ts`; newest-trace lookup ~700ms -> ~0.7ms warm (>1000x); index build ~1s per 100k rows. - Live sandbox reproduction (2.0.15 build, prod-sized DB copy): daemon that previously never answered a single request in 150s served 200 OK within 6s of boot and kept serving. - Production cutover: pipeline.ready 60s+ -> <1s; health 200 OK @ 205ms; restart storm stopped (was 321 restarts that day, zero since). - tests/unit/storage/: 82/82 pass; tsc clean for storage/*. Commit-message-only note: no runtime code paths changed other than schema; the migration is additive and idempotent. --- .../migrations/013-traces-ts-index.sql | 26 +++++ .../core/storage/migrator.ts | 60 ++++++++-- .../tests/unit/storage/migrator.test.ts | 105 ++++++++++++++++++ 3 files changed, 184 insertions(+), 7 deletions(-) create mode 100644 apps/memos-local-plugin/core/storage/migrations/013-traces-ts-index.sql diff --git a/apps/memos-local-plugin/core/storage/migrations/013-traces-ts-index.sql b/apps/memos-local-plugin/core/storage/migrations/013-traces-ts-index.sql new file mode 100644 index 000000000..4a2e099c9 --- /dev/null +++ b/apps/memos-local-plugin/core/storage/migrations/013-traces-ts-index.sql @@ -0,0 +1,26 @@ +-- Speed up unfiltered newest-first trace reads (health endpoint + boot replay). +-- +-- `latestTraceTs()` runs `SELECT ... FROM traces ORDER BY ts DESC, id DESC +-- LIMIT 1` several times per `/api/v1/health` request, and the pipeline's +-- recent-events replay issues the same shape at bootstrap. Every existing +-- `traces` index leads with `owner_*`, `share_scope`, `session_id`, or +-- `episode_id` -- none is usable for an UNFILTERED newest-first read, so each +-- call degenerated into a full table scan plus a temp B-tree sort +-- (`SCAN traces` / `USE TEMP B-TREE FOR ORDER BY`). Because better-sqlite3 +-- executes statements synchronously on the JS event loop, that scan blocks +-- the whole daemon: HTTP connections were accepted by the kernel backlog but +-- never answered while it ran. +-- +-- Observed impact on an install whose `traces` table reached ~30k rows +-- (~235 MB with embedding blobs and tool-call JSON): boot took 40-60s of +-- near-100% CPU inside the scan, and every liveness probe timed out before +-- the daemon could answer -- so a 60s watchdog restarted the daemon roughly +-- every 3 minutes, forever (300+ restarts/day). Each doomed generation +-- re-ran the scan at boot and was killed mid-scan, making the storm +-- self-sustaining. +-- +-- A bare `(ts DESC, id DESC)` index turns the lookup into a single index +-- seek: ~0.7ms warm instead of ~700ms warm / tens-of-seconds cold. Build cost +-- is ~1s per 100k rows and `IF NOT EXISTS` keeps re-application free. + +CREATE INDEX IF NOT EXISTS idx_traces_ts ON traces(ts DESC, id DESC); diff --git a/apps/memos-local-plugin/core/storage/migrator.ts b/apps/memos-local-plugin/core/storage/migrator.ts index 47e42807c..270a2532c 100644 --- a/apps/memos-local-plugin/core/storage/migrator.ts +++ b/apps/memos-local-plugin/core/storage/migrator.ts @@ -90,7 +90,7 @@ function assertMonotonic(files: MigrationFile[]): void { export function runMigrations(db: StorageDb, dir: string = defaultMigrationsDir()): MigrationsResult { ensureSchemaMigrationsTable(db); const allFiles = discoverMigrations(dir); - const appliedVersions = getAppliedVersions(db); + const appliedNames = getAppliedMigrationNames(db); const applied: MigrationsResult["applied"] = []; let skipped = 0; @@ -102,22 +102,35 @@ export function runMigrations(db: StorageDb, dir: string = defaultMigrationsDir( // input, so turning unsafe mode on for the migration phase is safe. // `.unsafeMode()` may not be toggled inside a transaction, so we flip it // at the outer boundary. + const isPending = (file: MigrationFile): boolean => { + const recordedName = appliedNames.get(file.version); + // Pending when never recorded, or recorded under a FOREIGN name (release- + // train collision) for a migration whose apply path is safe to re-run. + if (recordedName === undefined) return true; + return recordedName !== file.name && REPAIRABLE_UNDER_NAME_COLLISION.has(file.version); + }; const needsUnsafe = allFiles.some( - (f) => !appliedVersions.has(f.version) && migrationNeedsUnsafeMode(f.fullPath), + (f) => isPending(f) && migrationNeedsUnsafeMode(f.fullPath), ); if (needsUnsafe) db.raw.unsafeMode(true); try { for (const file of allFiles) { - if (appliedVersions.has(file.version)) { + if (!isPending(file)) { skipped++; continue; } const t0 = now(); db.tx(() => { applyMigration(db, file); + // Upsert: when the version row existed under a foreign name (release- + // train collision), repair the bookkeeping to reflect what THIS train + // last ensured. When the row is fresh, this behaves like the plain + // INSERT it replaces. db.prepare( - `INSERT INTO schema_migrations (version, name, applied_at) VALUES (@version, @name, @applied_at)`, + `INSERT INTO schema_migrations (version, name, applied_at) + VALUES (@version, @name, @applied_at) + ON CONFLICT(version) DO UPDATE SET name = excluded.name, applied_at = excluded.applied_at`, ).run({ version: file.version, name: file.name, applied_at: now() }); }); const durationMs = now() - t0; @@ -202,6 +215,14 @@ function applyMigration(db: StorageDb, file: MigrationFile): void { } return; } + if (file.version === 13 && file.name === "traces-ts-index") { + // Same guard as 012: some test harnesses build partial schemas without a + // `traces` table; the index is meaningless there and must not fail boot. + if (tableExists(db, "traces")) { + db.exec(fs.readFileSync(file.fullPath, "utf8")); + } + return; + } db.exec(fs.readFileSync(file.fullPath, "utf8")); } @@ -400,13 +421,38 @@ function ensureSchemaMigrationsTable(db: StorageDb): void { ); } -function getAppliedVersions(db: StorageDb): Set { +/** + * Applied migrations keyed by version number, valued by name. + * + * The npm plugin train and the monorepo train have historically reused + * version numbers for DIFFERENT migrations (e.g. a database migrated by the + * other train carries `(13, '')`). Version-only bookkeeping + * makes such databases silently skip any same-numbered migration shipped by + * this build -- including additive repair migrations that are safe to apply. + */ +function getAppliedMigrationNames(db: StorageDb): Map { const rows = db - .prepare(`SELECT version FROM schema_migrations`) + .prepare( + `SELECT version, name FROM schema_migrations`, + ) .all(); - return new Set(rows.map((r) => r.version)); + return new Map(rows.map((r) => [r.version, r.name])); } +/** + * Migrations whose `applyMigration()` path is safe to execute even when the + * same version number was already recorded under a DIFFERENT name (see + * `getAppliedMigrationNames`). Every entry here is guarded: it either no-ops + * through existence checks (`ensureColumn`, `tableExists`, `CREATE ... IF NOT + * EXISTS`) or performs a strictly additive change, so running it against a + * database that already has the equivalent objects from another train is a + * cheap no-op rather than a corruption risk. Anything NOT listed here keeps + * the conservative behaviour: a version-number collision skips the file. + */ +const REPAIRABLE_UNDER_NAME_COLLISION = new Set([ + 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, +]); + /** * Convenience helper for tests / CLIs: open, migrate, return. */ diff --git a/apps/memos-local-plugin/tests/unit/storage/migrator.test.ts b/apps/memos-local-plugin/tests/unit/storage/migrator.test.ts index c0e0eb215..9e23e810b 100644 --- a/apps/memos-local-plugin/tests/unit/storage/migrator.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/migrator.test.ts @@ -205,4 +205,109 @@ describe("storage/migrator", () => { db.close(); } }); + + it("013-traces-ts-index creates the bare-ts index and repairs train-collision bookkeeping", () => { + // Regression test for the Aug 2026 restart storm: `latestTraceTs()` runs + // an unfiltered newest-first trace read several times per /api/v1/health + // request. No existing index leads with bare `ts`, so every call was a + // full scan + temp B-tree sort; on a large traces table that blocked the + // synchronous better-sqlite3 event loop long enough that health probes + // timed out and a liveness watchdog restart-looped the daemon forever. + // + // This test also exercises the release-train heal: databases migrated by + // the other train carry schema_migrations rows whose VERSION numbers + // collide with different NAMES (observed: rows 13/14 named + // skill-repair-origin / episode-outcome). Repairable additive migrations + // must still apply and repair their bookkeeping row. + const { dbPath, cleanup } = tmpDb(); + cleanups.push(cleanup); + const db = openDb({ filepath: dbPath, agent: "openclaw" }); + try { + // Simulate a database previously migrated by the OTHER release train: + // versions 13/14 exist under foreign names before this build runs. + db.exec(` + CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_at INTEGER NOT NULL + ) STRICT; + `); + db.exec( + `INSERT INTO schema_migrations (version, name, applied_at) VALUES (13, 'skill-repair-origin', 1), (14, 'episode-outcome', 1)`, + ); + + const result = runMigrations(db); + expect(result.applied.map((m) => m.name)).toContain("traces-ts-index"); + + // The index exists... + const index = db + .prepare( + `SELECT name FROM sqlite_master WHERE type='index' AND name='idx_traces_ts'`, + ) + .get(); + expect(index?.name).toBe("idx_traces_ts"); + + // ...the plan for newest-first reads uses it instead of a table scan, + // and the foreign bookkeeping row was repaired to THIS train's name. + const detail = db + .prepare( + `SELECT sql FROM sqlite_master WHERE type='index' AND name='idx_traces_ts'`, + ) + .get(); + expect(detail?.sql).toContain("ts DESC"); + const repaired = db + .prepare( + `SELECT name FROM schema_migrations WHERE version = 13`, + ) + .get(); + expect(repaired?.name).toBe("traces-ts-index"); + + // Re-running is idempotent: everything counts as skipped. + const again = runMigrations(db); + expect(again.applied).toHaveLength(0); + expect(again.skipped).toBe(again.total); + } finally { + db.close(); + } + }); + + it("keeps skipping a version recorded under a foreign name when that migration is not repairable", () => { + // Conservative path: only migrations in the repairable allowlist may run + // under a version/name collision. Everything else keeps the historical + // behaviour -- the foreign record wins and the file is skipped untouched. + const { dbPath, cleanup } = tmpDb(); + cleanups.push(cleanup); + const db = openDb({ filepath: dbPath, agent: "openclaw" }); + try { + db.exec(` + CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_at INTEGER NOT NULL + ) STRICT; + `); + db.exec( + `INSERT INTO schema_migrations (version, name, applied_at) VALUES (11, 'their-hub-sharing-renamed', 1)`, + ); + + const result = runMigrations(db); + expect(result.applied.map((m) => m.version)).not.toContain(11); + // The hub-sharing objects were NOT created because 011 was skipped... + const hubTable = db + .prepare( + `SELECT name FROM sqlite_master WHERE type='table' AND name='hub_shared_skills'`, + ) + .get(); + expect(hubTable).toBeUndefined(); + // ...and the foreign bookkeeping row is preserved verbatim. + const row = db + .prepare( + `SELECT name FROM schema_migrations WHERE version = 11`, + ) + .get(); + expect(row?.name).toBe("their-hub-sharing-renamed"); + } finally { + db.close(); + } + }); }); From 7b35fcd8dff04108a0c684e0e909ef60a7f3d6a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Wed, 26 Aug 2026 16:38:21 +0800 Subject: [PATCH 29/34] fix(plugin): harden crystallizer draft recovery --- .../core/skill/crystallize.ts | 308 ++++++++++++++++-- .../unit/skill/crystallize-validator.test.ts | 15 +- .../tests/unit/skill/crystallize.test.ts | 168 +++++++++- .../unit/skill/skill.integration.test.ts | 22 ++ 4 files changed, 470 insertions(+), 43 deletions(-) diff --git a/apps/memos-local-plugin/core/skill/crystallize.ts b/apps/memos-local-plugin/core/skill/crystallize.ts index 17e9ad146..04de81654 100644 --- a/apps/memos-local-plugin/core/skill/crystallize.ts +++ b/apps/memos-local-plugin/core/skill/crystallize.ts @@ -9,7 +9,7 @@ * traces we fail fast with `skipped_reason="no-evidence"`. */ -import type { LlmClient, LlmMessage } from "../llm/types.js"; +import type { LlmClient, LlmJsonCompletion, LlmMessage } from "../llm/types.js"; import { detectModelRefusal } from "../llm/refusal.js"; import { detectDominantLanguage, @@ -24,7 +24,7 @@ import { sanitizeDerivedText, } from "../safety/content.js"; import type { EpisodeId, PolicyRow, SkillRow, TraceRow } from "../types.js"; -import { MemosError } from "../../agent-contract/errors.js"; +import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; import { extractToolNames } from "./tool-names.js"; import type { SkillModelRefusalDetails, @@ -69,6 +69,48 @@ export type CrystallizeResult = | { ok: true; draft: SkillCrystallizationDraft } | { ok: false; skippedReason: string; modelRefusal?: SkillModelRefusalDetails }; +interface DraftShapeDiagnostics { + rootType: string; + presentFields: string[]; + unknownFieldCount: number; + summaryType: string; + stepsType: string; + rawStepCount: number | null; + normalisedStepCount: number; +} + +interface PreparedDraft { + draft: SkillCrystallizationDraft; + shape: DraftShapeDiagnostics; + repairedFields: Array<"summary" | "steps">; + repairSources: Partial>; + usedAliases: string[]; +} + +const KNOWN_DRAFT_FIELDS = new Set([ + "name", + "display_title", + "displayTitle", + "summary", + "description", + "parameters", + "preconditions", + "steps", + "procedure", + "examples", + "tags", + "tools", + "decision_guidance", + "decisionGuidance", +]); + +const SKILL_DRAFT_SCHEMA_HINT = `{ + "name": "snake_case string", + "display_title": "string", + "summary": "string", + "steps": [{ "title": "string", "body": "string" }] +}`; + /** * Run one crystallization call and return a normalised draft. */ @@ -122,7 +164,8 @@ export async function crystallizeDraft( op: "skill.crystallize", phase: "skill", episodeId: input.episodeId, - schemaHint: "skill-crystallize.v2", + schemaHint: SKILL_DRAFT_SCHEMA_HINT, + malformedRetries: 0, }, ); const rawRefusal = detectModelRefusal(rsp.raw); @@ -139,7 +182,9 @@ export async function crystallizeDraft( }); return { ok: false, skippedReason: "llm-refusal", modelRefusal }; } - const draft = normaliseDraft(rsp.value, input); + const prepared = prepareDraftFromResponse(rsp, input, log, "initial"); + logDraftShape(prepared, rsp, input, log, "initial"); + const draft = prepared.draft; const draftRefusal = detectModelRefusal(draft); if (draftRefusal) { const modelRefusal = { @@ -154,7 +199,7 @@ export async function crystallizeDraft( }); return { ok: false, skippedReason: "llm-refusal", modelRefusal }; } - if (deps.validate) deps.validate(draft); + validatePreparedDraft(draft, prepared.shape, rsp, deps.validate, log, input, "initial"); return { ok: true, draft }; } catch (err) { const message = err instanceof Error ? err.message : String(err); @@ -202,7 +247,8 @@ The error was: ${message}. Please correct this and generate a valid JSON skill d op: "skill.crystallize", phase: "skill", episodeId: input.episodeId, - schemaHint: "skill-crystallize.v2", + schemaHint: SKILL_DRAFT_SCHEMA_HINT, + malformedRetries: 0, }, ); const retryRawRefusal = detectModelRefusal(rsp.raw); @@ -219,7 +265,9 @@ The error was: ${message}. Please correct this and generate a valid JSON skill d }); return { ok: false, skippedReason: "llm-refusal", modelRefusal }; } - const draft = normaliseDraft(rsp.value, input); + const prepared = prepareDraftFromResponse(rsp, input, log, "retry"); + logDraftShape(prepared, rsp, input, log, "retry"); + const draft = prepared.draft; const draftRefusal = detectModelRefusal(draft); if (draftRefusal) { const modelRefusal = { @@ -234,7 +282,7 @@ The error was: ${message}. Please correct this and generate a valid JSON skill d }); return { ok: false, skippedReason: "llm-refusal", modelRefusal }; } - if (deps.validate) deps.validate(draft); + validatePreparedDraft(draft, prepared.shape, rsp, deps.validate, log, input, "retry"); log.warn("skill.crystallize.retry_succeeded", { policyId: input.policy.id, error: message, @@ -255,6 +303,196 @@ The error was: ${message}. Please correct this and generate a valid JSON skill d } } +function prepareDraftFromResponse( + rsp: LlmJsonCompletion>, + input: CrystallizeInput, + log: Logger, + attempt: "initial" | "retry", +): PreparedDraft { + try { + return prepareDraft(rsp.value, input); + } catch (err) { + const shape = emptyDraftShape(rsp.value); + log.warn("skill.crystallize.shape_invalid", { + policyId: input.policy.id, + provider: rsp.provider, + model: rsp.model, + attempt, + reason: "invalid-root", + shape, + }); + const message = err instanceof Error ? err.message : String(err); + throw new MemosError(ERROR_CODES.LLM_OUTPUT_MALFORMED, message, { + provider: rsp.provider, + rawPreview: rsp.raw.slice(0, 512), + shape, + }); + } +} + +function prepareDraft(raw: unknown, input: CrystallizeInput): PreparedDraft { + if (!isRecord(raw)) { + throw new Error("skill.crystallize.invalid: root must be an object"); + } + + const draft = normaliseDraft(raw, input); + const shape = inspectDraftShape(raw, draft.steps.length); + const repairedFields: PreparedDraft["repairedFields"] = []; + const repairSources: PreparedDraft["repairSources"] = {}; + const usedAliases: string[] = []; + + if (!cleanOptionalText(raw.summary) && cleanOptionalText(raw.description)) { + usedAliases.push("description->summary"); + } + if (!Array.isArray(raw.steps) && raw.procedure !== undefined) { + usedAliases.push("procedure->steps"); + } + if (Array.isArray(raw.steps) && raw.steps.some(stepUsesAlias)) { + usedAliases.push("step-aliases"); + } + + if (!draft.summary) { + const candidates: Array<[string, unknown]> = [ + ["step.body", draft.steps[0]?.body], + ["step.title", draft.steps[0]?.title], + ["displayTitle", draft.displayTitle], + ["name", draft.name], + ]; + const source = candidates.find(([, value]) => sanitizeDerivedText(value)); + if (source) { + draft.summary = sanitizeDerivedText(source[1]).slice(0, 200); + repairedFields.push("summary"); + repairSources.summary = source[0]; + } + } + + if (draft.steps.length === 0) { + const policyBody = sanitizeDerivedMarkdown(input.policy.procedure).slice(0, 2000); + if (policyBody) { + const policyTitle = sanitizeDerivedText(input.policy.title || input.policy.trigger) + .slice(0, 200); + draft.steps = [{ + title: policyTitle || sanitizeDerivedText(policyBody).slice(0, 32), + body: policyBody, + }]; + repairedFields.push("steps"); + repairSources.steps = "policy.procedure"; + } + } + + return { draft, shape, repairedFields, repairSources, usedAliases }; +} + +function logDraftShape( + prepared: PreparedDraft, + rsp: LlmJsonCompletion>, + input: CrystallizeInput, + log: Logger, + attempt: "initial" | "retry", +): void { + const common = { + policyId: input.policy.id, + provider: rsp.provider, + model: rsp.model, + attempt, + shape: prepared.shape, + }; + if (prepared.repairedFields.length > 0) { + log.warn("skill.crystallize.shape_repaired", { + ...common, + repairedFields: prepared.repairedFields, + repairSources: prepared.repairSources, + }); + } else if (prepared.usedAliases.length > 0) { + log.warn("skill.crystallize.shape_normalised", { + ...common, + aliases: prepared.usedAliases, + }); + } +} + +function validatePreparedDraft( + draft: SkillCrystallizationDraft, + shape: DraftShapeDiagnostics, + rsp: LlmJsonCompletion>, + validate: CrystallizeDeps["validate"], + log: Logger, + input: CrystallizeInput, + attempt: "initial" | "retry", +): void { + try { + defaultDraftValidator(draft); + if (validate && validate !== defaultDraftValidator) validate(draft); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log.warn("skill.crystallize.shape_invalid", { + policyId: input.policy.id, + provider: rsp.provider, + model: rsp.model, + attempt, + reason: safeValidationReason(message), + shape, + }); + throw new MemosError(ERROR_CODES.LLM_OUTPUT_MALFORMED, message, { + provider: rsp.provider, + rawPreview: rsp.raw.slice(0, 512), + shape, + }); + } +} + +function safeValidationReason(message: string): string { + if (message.endsWith("missing name")) return "missing-name"; + if (message.endsWith("missing summary")) return "missing-summary"; + if (message.endsWith("missing steps")) return "missing-steps"; + return "validator-rejected"; +} + +function inspectDraftShape( + raw: Record, + normalisedStepCount: number, +): DraftShapeDiagnostics { + const keys = Object.keys(raw); + return { + rootType: "object", + presentFields: keys.filter((key) => KNOWN_DRAFT_FIELDS.has(key)).sort(), + unknownFieldCount: keys.filter((key) => !KNOWN_DRAFT_FIELDS.has(key)).length, + summaryType: valueType(raw.summary), + stepsType: valueType(raw.steps), + rawStepCount: Array.isArray(raw.steps) ? raw.steps.length : null, + normalisedStepCount, + }; +} + +function emptyDraftShape(raw: unknown): DraftShapeDiagnostics { + return { + rootType: valueType(raw), + presentFields: [], + unknownFieldCount: 0, + summaryType: "unavailable", + stepsType: "unavailable", + rawStepCount: null, + normalisedStepCount: 0, + }; +} + +function valueType(value: unknown): string { + if (value === null) return "null"; + if (Array.isArray(value)) return "array"; + return typeof value; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function stepUsesAlias(value: unknown): boolean { + if (typeof value === "string") return true; + if (!isRecord(value)) return false; + return value.name !== undefined || value.description !== undefined || + value.content !== undefined || value.instruction !== undefined; +} + function rawPreviewFromError(err: unknown): string | null { if (err instanceof MemosError && typeof err.details?.rawPreview === "string") { return err.details.rawPreview; @@ -353,11 +591,11 @@ function normaliseDraft( const displayTitle = sanitizeDerivedText(raw.display_title ?? raw.displayTitle ?? input.policy.title ?? name) || name; - const summary = sanitizeDerivedText(raw.summary); + const summary = cleanOptionalText(raw.summary) || cleanOptionalText(raw.description); const parameters = asArray(raw.parameters).map(coerceParameter).filter(Boolean) as SkillParameterDraft[]; const preconditions = sanitizeDerivedMarkdownList(asStringArray(raw.preconditions)); - const steps = asArray(raw.steps).map(coerceStep).filter(Boolean) as SkillStepDraft[]; + const steps = selectRawSteps(raw).map(coerceStep).filter(Boolean) as SkillStepDraft[]; const examples = asArray(raw.examples).map(coerceExample).filter(Boolean) as SkillExampleDraft[]; const tags = dedupeLc(sanitizeDerivedList(asStringArray(raw.tags))); // V7 §2.4.6 — coerce both `decision_guidance` (preferred LLM key) @@ -381,6 +619,22 @@ function normaliseDraft( }; } +function selectRawSteps(raw: Record): unknown[] { + if (Array.isArray(raw.steps)) return raw.steps; + if (typeof raw.steps === "string" && raw.steps.trim()) return [raw.steps]; + if (Array.isArray(raw.procedure)) return raw.procedure; + if (typeof raw.procedure === "string" && raw.procedure.trim()) return [raw.procedure]; + return []; +} + +function cleanOptionalText(value: unknown): string { + return typeof value === "string" ? sanitizeDerivedText(value) : ""; +} + +function cleanOptionalMarkdown(value: unknown): string { + return typeof value === "string" ? sanitizeDerivedMarkdown(value) : ""; +} + function coerceDecisionGuidance(raw: unknown): { preference: string[]; antiPattern: string[]; @@ -446,10 +700,18 @@ function coerceParameter(x: unknown): SkillParameterDraft | null { } function coerceStep(x: unknown): SkillStepDraft | null { + if (typeof x === "string") { + const body = sanitizeDerivedMarkdown(x); + if (!body) return null; + return { title: sanitizeDerivedText(body).slice(0, 32), body }; + } if (!x || typeof x !== "object") return null; const o = x as Record; - const title = sanitizeDerivedText(o.title); - const body = sanitizeDerivedMarkdown(o.body); + const title = cleanOptionalText(o.title) || cleanOptionalText(o.name); + const body = cleanOptionalMarkdown(o.body) || + cleanOptionalMarkdown(o.description) || + cleanOptionalMarkdown(o.content) || + cleanOptionalMarkdown(o.instruction); if (!title && !body) return null; return { title: title || body.slice(0, 32), body }; } @@ -469,12 +731,10 @@ function capString(s: string, cap: number): string { } /** - * A sensible default validator used both in production and in tests. - * Throws only when the draft is structurally unusable (no name). Missing - * summary/steps are repaired from the remaining fields instead — LLMs (e.g. - * deepseek-v4-flash) routinely return valid JSON drafts that omit `summary` - * or the `steps` array, and rejecting those stalls the crystallizer queue - * (see issue #2143). + * A sensible default validator used both in production and in tests. Summary + * can be recovered from the draft itself, but steps must already contain + * actionable material. The runtime normaliser may ground missing steps in the + * source policy; this validator never invents a generic procedure. */ export function defaultDraftValidator(draft: SkillCrystallizationDraft): void { if (!draft.name) throw new Error("skill.crystallize.invalid: missing name"); @@ -486,18 +746,10 @@ export function defaultDraftValidator(draft: SkillCrystallizationDraft): void { draft.steps?.[0]?.body || draft.steps?.[0]?.title || draft.displayTitle || - draft.name || - "skill procedure"; + draft.name; draft.summary = autoSummary.slice(0, 200); } if (!draft.steps || draft.steps.length === 0) { - // Auto-generate a single step when the LLM omits the steps array - // (mirror of the summary fallback chain above). - const autoBody = draft.summary || draft.displayTitle || draft.name || ""; - if (autoBody) { - draft.steps = [{ title: "Execute the fix", body: autoBody.slice(0, 2000) }]; - } else { - throw new Error("skill.crystallize.invalid: missing steps"); - } + throw new Error("skill.crystallize.invalid: missing steps"); } } diff --git a/apps/memos-local-plugin/tests/unit/skill/crystallize-validator.test.ts b/apps/memos-local-plugin/tests/unit/skill/crystallize-validator.test.ts index 348a2ac72..54a2d1739 100644 --- a/apps/memos-local-plugin/tests/unit/skill/crystallize-validator.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/crystallize-validator.test.ts @@ -24,7 +24,7 @@ describe("defaultDraftValidator", () => { it("falls back through step title, displayTitle, then name for the summary", () => { const draft = makeDraft({ summary: "", steps: [] }); - defaultDraftValidator(draft); + expect(() => defaultDraftValidator(draft)).toThrow(/missing steps/); expect(draft.summary).toBe("Alpine pip install with system deps"); }); @@ -40,19 +40,14 @@ describe("defaultDraftValidator", () => { it("uses || not ?? — an empty-string summary still triggers the fallback", () => { const draft = makeDraft({ summary: "", steps: [] }); - defaultDraftValidator(draft); + expect(() => defaultDraftValidator(draft)).toThrow(/missing steps/); expect(draft.summary).not.toBe(""); }); - it("auto-generates a single step when the steps array is empty", () => { + it("rejects missing steps instead of inventing a generic procedure", () => { const draft = makeDraft({ steps: [] }); - defaultDraftValidator(draft); - expect(draft.steps).toEqual([ - { - title: "Execute the fix", - body: "Ensure system libs exist before pip install on alpine.", - }, - ]); + expect(() => defaultDraftValidator(draft)).toThrow(/missing steps/); + expect(draft.steps).toEqual([]); }); it("still rejects a draft with no name", () => { diff --git a/apps/memos-local-plugin/tests/unit/skill/crystallize.test.ts b/apps/memos-local-plugin/tests/unit/skill/crystallize.test.ts index bcaccaf67..3c6ec0d32 100644 --- a/apps/memos-local-plugin/tests/unit/skill/crystallize.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/crystallize.test.ts @@ -5,6 +5,7 @@ import { defaultDraftValidator, } from "../../../core/skill/crystallize.js"; import { rootLogger } from "../../../core/logger/index.js"; +import type { Logger } from "../../../core/logger/types.js"; import type { LlmClient, LlmJsonCompletion } from "../../../core/llm/types.js"; import type { PolicyRow, TraceRow } from "../../../core/types.js"; import { fakeLlm, throwingLlm } from "../../helpers/fake-llm.js"; @@ -59,6 +60,22 @@ function mkTrace(id: string, userText: string): TraceRow { const log = rootLogger.child({ channel: "core.skill.crystallize" }); +function loggerRecordingWarnings( + warnings: Array<{ message: string; data?: Record }>, +): Logger { + return new Proxy(log, { + get(target, prop, receiver) { + if (prop === "warn") { + return (message: string, data?: Record) => { + warnings.push({ message, data }); + }; + } + const value = Reflect.get(target, prop, receiver) as unknown; + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} + function refusalLlm(raw: string): LlmClient { return { ...fakeLlm(), @@ -156,7 +173,7 @@ describe("skill/crystallize", () => { const r = await crystallizeDraft( { policy, evidence: [mkTrace("tr_1", "pip fails")], namingSpace: [] }, - { llm, log, config: makeSkillConfig(), validate: defaultDraftValidator }, + { llm, log, config: makeSkillConfig() }, ); expect(r.ok).toBe(true); @@ -230,10 +247,10 @@ describe("skill/crystallize", () => { expect(r.modelRefusal?.content).toContain("I cannot process this request"); }); - it("repairs drafts the strict validator would have rejected (issue #2143)", async () => { + it("repairs missing summary and steps from grounded policy fields (issue #2143)", async () => { // A draft with an empty summary AND no steps used to be rejected with - // skill.crystallize.invalid: missing summary / missing steps. The lenient - // validator auto-generates both, so the same draft now crystallizes. + // skill.crystallize.invalid: missing summary / missing steps. The runtime + // now repairs both from grounded draft/policy fields before validation. const llm = fakeLlm({ completeJson: { "skill.crystallize": makeDraft({ steps: [], summary: "" }) as unknown, @@ -246,7 +263,148 @@ describe("skill/crystallize", () => { expect(r.ok).toBe(true); if (r.ok) { expect(r.draft.summary).not.toBe(""); - expect(r.draft.steps.length).toBeGreaterThan(0); + expect(r.draft.steps).toEqual([ + { + title: "install system libs before pip", + body: "1. detect 2. apk add 3. retry", + }, + ]); + expect(r.draft.steps[0]!.title).not.toBe("Execute the fix"); } }); + + it("normalises only explicit summary and step aliases", async () => { + const llm = fakeLlm({ + completeJson: { + "skill.crystallize": { + ...makeDraft(), + summary: " ", + description: "Install Alpine dependencies before retrying pip.", + steps: [ + "Inspect the pip error", + { + title: " ", + name: "Install packages", + body: " ", + instruction: "Run apk add for the missing libraries", + }, + ], + }, + }, + }); + + const r = await crystallizeDraft( + { policy: mkPolicy(), evidence: [mkTrace("tr_1", "pip fails")], namingSpace: [] }, + { llm, log, config: makeSkillConfig(), validate: defaultDraftValidator }, + ); + + expect(r.ok).toBe(true); + if (!r.ok) return; + expect(r.draft.summary).toBe("Install Alpine dependencies before retrying pip."); + expect(r.draft.steps).toEqual([ + { title: "Inspect the pip error", body: "Inspect the pip error" }, + { title: "Install packages", body: "Run apk add for the missing libraries" }, + ]); + }); + + it("retries once when the draft has no steps and policy has no grounded procedure", async () => { + let calls = 0; + const llm = fakeLlm({ + completeJson: { + "skill.crystallize": () => { + calls += 1; + return calls === 1 ? makeDraft({ steps: [] }) : makeDraft(); + }, + }, + }); + const policy = { ...mkPolicy(), procedure: "" }; + + const r = await crystallizeDraft( + { policy, evidence: [mkTrace("tr_1", "pip fails")], namingSpace: [] }, + { llm, log, config: makeSkillConfig() }, + ); + + expect(r.ok).toBe(true); + expect(calls).toBe(2); + }); + + it("retries once when the parsed JSON root has the wrong shape", async () => { + let calls = 0; + const llm = fakeLlm({ + completeJson: { + "skill.crystallize": () => { + calls += 1; + return calls === 1 ? "not-an-object" : makeDraft(); + }, + }, + }); + + const r = await crystallizeDraft( + { policy: mkPolicy(), evidence: [mkTrace("tr_1", "pip fails")], namingSpace: [] }, + { llm, log, config: makeSkillConfig(), validate: defaultDraftValidator }, + ); + + expect(r.ok).toBe(true); + expect(calls).toBe(2); + }); + + it("rejects after one retry when neither response nor policy has procedure steps", async () => { + let calls = 0; + const llm = fakeLlm({ + completeJson: { + "skill.crystallize": () => { + calls += 1; + return makeDraft({ steps: [] }); + }, + }, + }); + const policy = { ...mkPolicy(), procedure: "" }; + + const r = await crystallizeDraft( + { policy, evidence: [mkTrace("tr_1", "pip fails")], namingSpace: [] }, + { llm, log, config: makeSkillConfig(), validate: defaultDraftValidator }, + ); + + expect(r.ok).toBe(false); + if (r.ok) return; + expect(r.skippedReason).toMatch(/missing steps/); + expect(calls).toBe(2); + }); + + it("logs only shape metadata when repairing a malformed draft", async () => { + const warnings: Array<{ message: string; data?: Record }> = []; + const sensitive = "SENSITIVE-CONTENT-MUST-NOT-BE-LOGGED"; + const llm = fakeLlm({ + completeJson: { + "skill.crystallize": { + ...makeDraft({ steps: [] }), + unexpected: sensitive, + }, + }, + }); + + const r = await crystallizeDraft( + { policy: mkPolicy(), evidence: [mkTrace("tr_1", "pip fails")], namingSpace: [] }, + { + llm, + log: loggerRecordingWarnings(warnings), + config: makeSkillConfig(), + validate: defaultDraftValidator, + }, + ); + + expect(r.ok).toBe(true); + const shapeLog = warnings.find((entry) => entry.message === "skill.crystallize.shape_repaired"); + expect(shapeLog?.data).toMatchObject({ + repairedFields: ["steps"], + shape: { + summaryType: "string", + stepsType: "array", + rawStepCount: 0, + normalisedStepCount: 0, + unknownFieldCount: 1, + }, + }); + expect(JSON.stringify(shapeLog)).not.toContain(sensitive); + }); }); diff --git a/apps/memos-local-plugin/tests/unit/skill/skill.integration.test.ts b/apps/memos-local-plugin/tests/unit/skill/skill.integration.test.ts index d2280fdc9..476134abc 100644 --- a/apps/memos-local-plugin/tests/unit/skill/skill.integration.test.ts +++ b/apps/memos-local-plugin/tests/unit/skill/skill.integration.test.ts @@ -107,6 +107,28 @@ describe("skill/runSkill (integration)", () => { expect(all[0]!.sourcePolicyIds).toContain(policyId); }); + it("persists grounded policy steps when the LLM omits its steps", async () => { + const h = open(); + const { policyId } = seedFullCandidate(h); + const policy = h.repos.policies.getById(policyId)!; + const { deps } = makeDeps(h, { + llm: fakeLlm({ + completeJson: { + "skill.crystallize": makeDraft({ steps: [] }), + }, + }), + }); + + const r = await runSkill({ trigger: "manual", policyId }, deps); + + expect(r.crystallized).toBe(1); + const stored = h.repos.skills.list()[0]!; + expect(stored.procedureJson?.steps).toEqual([ + { title: policy.title, body: policy.procedure }, + ]); + expect(stored.invocationGuide).not.toContain("Execute the fix"); + }); + it("rebuilds an existing skill when the policy has drifted", async () => { const h = open(); const { policyId } = seedFullCandidate(h); From 319769188b32f2d7a5da1bd33327576c7653aff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Wed, 26 Aug 2026 17:10:48 +0800 Subject: [PATCH 30/34] test(plugin): format Hermes UTF-8 regression --- .../tests/python/test_hermes_provider_pipeline.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index 73359cf77..ea6818ca9 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -1150,9 +1150,7 @@ def test_memos_get_world_model_returns_utf8_chinese(self) -> None: bridge = ChineseToolResultBridge() provider = self._make_provider(bridge) - raw = provider.handle_tool_call( - "memos_get", {"id": "world-cn-1", "kind": "world_model"} - ) + raw = provider.handle_tool_call("memos_get", {"id": "world-cn-1", "kind": "world_model"}) self.assertIn(ChineseToolResultBridge._CH_WORLD_TITLE, raw) parsed = json.loads(raw) From 068a701aa4cc7aed557cdec64a920469a1ae2b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Wed, 26 Aug 2026 17:45:01 +0800 Subject: [PATCH 31/34] fix(plugin): reconcile stale Hermes bridge status --- apps/memos-local-plugin/bridge.cts | 157 ++------------ apps/memos-local-plugin/bridge.mts | 165 ++------------ apps/memos-local-plugin/bridge/status.ts | 186 ++++++++++++++++ .../tests/unit/bridge/status.test.ts | 203 ++++++++++++++++++ 4 files changed, 425 insertions(+), 286 deletions(-) create mode 100644 apps/memos-local-plugin/bridge/status.ts create mode 100644 apps/memos-local-plugin/tests/unit/bridge/status.test.ts diff --git a/apps/memos-local-plugin/bridge.cts b/apps/memos-local-plugin/bridge.cts index 5e533779e..f348b8c12 100644 --- a/apps/memos-local-plugin/bridge.cts +++ b/apps/memos-local-plugin/bridge.cts @@ -33,9 +33,6 @@ const { homedir } = require("node:os") as typeof import("node:os"); // eslint-disable-next-line @typescript-eslint/no-require-imports const url = require("node:url") as typeof import("node:url"); -const BRIDGE_STATUS_HEARTBEAT_MS = 5_000; -const BRIDGE_STATUS_STALE_MS = 20_000; -const BRIDGE_STATUS_FILE = "bridge-status.json"; // If core.shutdown() or waitForShutdown() blocks (e.g. L2/L3 LLM calls // hanging during flush), the bridge process would never exit after stdin // EOF or SIGTERM. Race against this deadline so the process always exits @@ -61,15 +58,6 @@ interface BridgeArgs { runtimeScope?: string; } -type BridgeStatus = "connected" | "reconnecting" | "disconnected" | "unknown"; - -interface BridgeStatusSnapshot { - status: BridgeStatus; - lastOkAt: number | null; - lastErrorAt: number | null; - lastError: string | null; -} - function parseArgs(argv: readonly string[]): BridgeArgs { const args: BridgeArgs = { daemon: false, noViewer: false, agent: "openclaw" }; for (const raw of argv) { @@ -218,6 +206,13 @@ async function main(): Promise { const { isHermesChatRunning } = (await importEsm( runtimeModule("bridge/hermes-process.ts", "dist/bridge/hermes-process.js") )) as typeof import("./bridge/hermes-process.js"); + const { + BRIDGE_STATUS_FILE, + createBridgeStatusReader, + createBridgeStatusWriter, + } = (await importEsm( + runtimeModule("bridge/status.ts", "dist/bridge/status.js") + )) as typeof import("./bridge/status.js"); const rootDir = pluginRoot(); const pkgVersion = JSON.parse( @@ -339,13 +334,15 @@ async function main(): Promise { (core as { bindTelemetry?: (t: InstanceType) => void }).bindTelemetry?.(telemetry); telemetry.trackPluginStarted(args.agent); + const bridgeStatusFile = path.join(home.root, BRIDGE_STATUS_FILE); + const bridgeStatusWriter = + args.agent === "hermes" && !args.daemon + ? createBridgeStatusWriter(bridgeStatusFile) + : null; const bridgeStatus = args.agent === "hermes" - ? createBridgeStatusTracker( - path.join(home.root, BRIDGE_STATUS_FILE), - args.daemon, - isHermesChatRunning, - ) + ? bridgeStatusWriter ?? + createBridgeStatusReader(bridgeStatusFile, { isHermesChatRunning }) : null; // Process-level error reporting. Without these handlers a crash in @@ -391,7 +388,7 @@ async function main(): Promise { const viewerPort = AGENT_DEFAULT_PORTS[args.agent]; let bridgeHeartbeat: - | ReturnType["startHeartbeat"]> + | ReturnType["startHeartbeat"]> | undefined; // ─── Startup ordering invariant (issue #1747 + host LLM fallback) ─── @@ -426,11 +423,11 @@ async function main(): Promise { // `tests/unit/bridge/bridge-startup-ordering.test.ts`. if (!args.daemon) { stdio = startStdioServer({ core }); - bridgeStatus?.markConnected(); - bridgeHeartbeat = bridgeStatus?.startHeartbeat(); + bridgeStatusWriter?.markConnected(); + bridgeHeartbeat = bridgeStatusWriter?.startHeartbeat(); void stdio.done.then(() => { bridgeHeartbeat?.stop(); - bridgeStatus?.markDisconnected("Hermes chat disconnected"); + bridgeStatusWriter?.markDisconnected("Hermes chat disconnected"); }); } @@ -672,124 +669,6 @@ function classifyErrorCode(err: unknown): string { return "unknown"; } -function createBridgeStatusTracker( - statusFile: string, - daemon: boolean, - isHermesChatRunning: () => boolean, -): { - snapshot(): BridgeStatusSnapshot; - markConnected(): void; - markDisconnected(message: string): void; - startHeartbeat(): { stop(): void }; -} { - let snapshot: BridgeStatusSnapshot = daemon - ? { - status: "disconnected", - lastOkAt: null, - lastErrorAt: Date.now(), - lastError: "Hermes chat is not connected", - } - : { - status: "unknown", - lastOkAt: null, - lastErrorAt: null, - lastError: null, - }; - - function writeStatus(next: BridgeStatusSnapshot): void { - snapshot = next; - try { - fs.mkdirSync(path.dirname(statusFile), { recursive: true }); - fs.writeFileSync(statusFile, JSON.stringify(next), "utf8"); - } catch { - // Status display must never affect chat capture. - } - } - - function readStatus(): BridgeStatusSnapshot | null { - try { - const parsed = JSON.parse(fs.readFileSync(statusFile, "utf8")) as Partial; - if ( - parsed.status === "connected" || - parsed.status === "reconnecting" || - parsed.status === "disconnected" || - parsed.status === "unknown" - ) { - return { - status: parsed.status, - lastOkAt: typeof parsed.lastOkAt === "number" ? parsed.lastOkAt : null, - lastErrorAt: typeof parsed.lastErrorAt === "number" ? parsed.lastErrorAt : null, - lastError: typeof parsed.lastError === "string" ? parsed.lastError : null, - }; - } - } catch { - // Missing or corrupt status files are treated as disconnected. - } - return null; - } - - function applyStaleRule(raw: BridgeStatusSnapshot): BridgeStatusSnapshot { - if (raw.status === "disconnected" && daemon && isHermesChatRunning()) { - return { - status: "reconnecting", - lastOkAt: raw.lastOkAt, - lastErrorAt: raw.lastErrorAt, - lastError: "Hermes chat is running; waiting for memory bridge", - }; - } - if ( - raw.status === "connected" && - raw.lastOkAt != null && - Date.now() - raw.lastOkAt > BRIDGE_STATUS_STALE_MS - ) { - return { - status: "disconnected", - lastOkAt: raw.lastOkAt, - lastErrorAt: Date.now(), - lastError: "Hermes bridge heartbeat is stale", - }; - } - return raw; - } - - function markConnected(): void { - writeStatus({ - status: "connected", - lastOkAt: Date.now(), - lastErrorAt: snapshot.lastErrorAt, - lastError: snapshot.lastError, - }); - } - - function markDisconnected(message: string): void { - writeStatus({ - status: "disconnected", - lastOkAt: snapshot.lastOkAt, - lastErrorAt: Date.now(), - lastError: message, - }); - } - - return { - snapshot() { - return { ...applyStaleRule(readStatus() ?? snapshot) }; - }, - markConnected, - markDisconnected, - startHeartbeat() { - const timer = setInterval(() => { - markConnected(); - }, BRIDGE_STATUS_HEARTBEAT_MS); - (timer as unknown as { unref?: () => void }).unref?.(); - return { - stop() { - clearInterval(timer); - }, - }; - }, - }; -} - void main().catch((err) => { const detail = err instanceof Error ? err.stack ?? err.message : String(err); process.stderr.write( diff --git a/apps/memos-local-plugin/bridge.mts b/apps/memos-local-plugin/bridge.mts index d271fc7f2..75f7e7c37 100644 --- a/apps/memos-local-plugin/bridge.mts +++ b/apps/memos-local-plugin/bridge.mts @@ -28,18 +28,21 @@ * still work). There's no port-sharing or auto-promotion logic — * each agent has its own bookmarkable URL. */ -import * as childProcess from "node:child_process"; import * as fs from "node:fs"; import { homedir } from "node:os"; import * as path from "node:path"; import { fileURLToPath } from "node:url"; +import { isHermesChatRunning } from "./bridge/hermes-process.js"; +import { + BRIDGE_STATUS_FILE, + createBridgeStatusReader, + createBridgeStatusWriter, +} from "./bridge/status.js"; + const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); -const BRIDGE_STATUS_HEARTBEAT_MS = 5_000; -const BRIDGE_STATUS_STALE_MS = 20_000; -const BRIDGE_STATUS_FILE = "bridge-status.json"; // Keep both executable bridge entries within the same process-level budget. // Core shutdown has its own cooperative recovery cancellation, while this // outer deadline guarantees a broken provider cannot orphan the bridge. @@ -63,15 +66,6 @@ interface BridgeArgs { home?: string; } -type BridgeStatus = "connected" | "reconnecting" | "disconnected" | "unknown"; - -interface BridgeStatusSnapshot { - status: BridgeStatus; - lastOkAt: number | null; - lastErrorAt: number | null; - lastError: string | null; -} - function parseArgs(argv: readonly string[]): BridgeArgs { const args: BridgeArgs = { daemon: false, noViewer: false, agent: "openclaw" }; for (const raw of argv) { @@ -282,12 +276,15 @@ async function main(): Promise { (core as { bindTelemetry?: (t: InstanceType) => void }).bindTelemetry?.(telemetry); telemetry.trackPluginStarted(args.agent); + const bridgeStatusFile = path.join(home.root, BRIDGE_STATUS_FILE); + const bridgeStatusWriter = + args.agent === "hermes" && !args.daemon + ? createBridgeStatusWriter(bridgeStatusFile) + : null; const bridgeStatus = args.agent === "hermes" - ? createBridgeStatusTracker( - path.join(home.root, BRIDGE_STATUS_FILE), - args.daemon, - ) + ? bridgeStatusWriter ?? + createBridgeStatusReader(bridgeStatusFile, { isHermesChatRunning }) : null; // Process-level error reporting. Without these handlers a crash in @@ -333,7 +330,7 @@ async function main(): Promise { const viewerPort = AGENT_DEFAULT_PORTS[args.agent]; let bridgeHeartbeat: - | ReturnType["startHeartbeat"]> + | ReturnType["startHeartbeat"]> | undefined; // In stdio mode the host fallback path is a reverse JSON-RPC request @@ -344,11 +341,11 @@ async function main(): Promise { // fallback has a transport instead of tripping the lazy bridge guard. if (!args.daemon) { stdio = startStdioServer({ core }); - bridgeStatus?.markConnected(); - bridgeHeartbeat = bridgeStatus?.startHeartbeat(); + bridgeStatusWriter?.markConnected(); + bridgeHeartbeat = bridgeStatusWriter?.startHeartbeat(); void stdio.done.then(() => { bridgeHeartbeat?.stop(); - bridgeStatus?.markDisconnected("Hermes chat disconnected"); + bridgeStatusWriter?.markDisconnected("Hermes chat disconnected"); }); } @@ -574,132 +571,6 @@ function classifyErrorCode(err: unknown): string { return "unknown"; } -function createBridgeStatusTracker(statusFile: string, daemon: boolean): { - snapshot(): BridgeStatusSnapshot; - markConnected(): void; - markDisconnected(message: string): void; - startHeartbeat(): { stop(): void }; -} { - let snapshot: BridgeStatusSnapshot = daemon - ? { - status: "disconnected", - lastOkAt: null, - lastErrorAt: Date.now(), - lastError: "Hermes chat is not connected", - } - : { - status: "unknown", - lastOkAt: null, - lastErrorAt: null, - lastError: null, - }; - - function writeStatus(next: BridgeStatusSnapshot): void { - snapshot = next; - try { - fs.mkdirSync(path.dirname(statusFile), { recursive: true }); - fs.writeFileSync(statusFile, JSON.stringify(next), "utf8"); - } catch { - // Status display must never affect chat capture. - } - } - - function readStatus(): BridgeStatusSnapshot | null { - try { - const parsed = JSON.parse(fs.readFileSync(statusFile, "utf8")) as Partial; - if ( - parsed.status === "connected" || - parsed.status === "reconnecting" || - parsed.status === "disconnected" || - parsed.status === "unknown" - ) { - return { - status: parsed.status, - lastOkAt: typeof parsed.lastOkAt === "number" ? parsed.lastOkAt : null, - lastErrorAt: typeof parsed.lastErrorAt === "number" ? parsed.lastErrorAt : null, - lastError: typeof parsed.lastError === "string" ? parsed.lastError : null, - }; - } - } catch { - // Missing or corrupt status files are treated as disconnected. - } - return null; - } - - function applyStaleRule(raw: BridgeStatusSnapshot): BridgeStatusSnapshot { - if (raw.status === "disconnected" && daemon && isHermesChatRunning()) { - return { - status: "reconnecting", - lastOkAt: raw.lastOkAt, - lastErrorAt: raw.lastErrorAt, - lastError: "Hermes chat is running; waiting for memory bridge", - }; - } - if ( - raw.status === "connected" && - raw.lastOkAt != null && - Date.now() - raw.lastOkAt > BRIDGE_STATUS_STALE_MS - ) { - return { - status: "disconnected", - lastOkAt: raw.lastOkAt, - lastErrorAt: Date.now(), - lastError: "Hermes bridge heartbeat is stale", - }; - } - return raw; - } - - function markConnected(): void { - writeStatus({ - status: "connected", - lastOkAt: Date.now(), - lastErrorAt: snapshot.lastErrorAt, - lastError: snapshot.lastError, - }); - } - - function markDisconnected(message: string): void { - writeStatus({ - status: "disconnected", - lastOkAt: snapshot.lastOkAt, - lastErrorAt: Date.now(), - lastError: message, - }); - } - - return { - snapshot() { - return { ...applyStaleRule(readStatus() ?? snapshot) }; - }, - markConnected, - markDisconnected, - startHeartbeat() { - const timer = setInterval(() => { - markConnected(); - }, BRIDGE_STATUS_HEARTBEAT_MS); - (timer as unknown as { unref?: () => void }).unref?.(); - return { - stop() { - clearInterval(timer); - }, - }; - }, - }; -} - -function isHermesChatRunning(): boolean { - try { - const out = childProcess.execFileSync("pgrep", ["-f", "hermes chat"], { - encoding: "utf8", - timeout: 1000, - }); - return out.trim().length > 0; - } catch { - return false; - } -} - void main().catch((err) => { const detail = err instanceof Error ? err.stack ?? err.message : String(err); process.stderr.write( diff --git a/apps/memos-local-plugin/bridge/status.ts b/apps/memos-local-plugin/bridge/status.ts new file mode 100644 index 000000000..f9f5097f8 --- /dev/null +++ b/apps/memos-local-plugin/bridge/status.ts @@ -0,0 +1,186 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; + +export const BRIDGE_STATUS_FILE = "bridge-status.json"; +export const BRIDGE_STATUS_HEARTBEAT_MS = 5_000; +export const BRIDGE_STATUS_STALE_MS = 20_000; + +export type BridgeStatus = + | "connected" + | "reconnecting" + | "disconnected" + | "unknown"; + +export interface BridgeStatusSnapshot { + status: BridgeStatus; + lastOkAt: number | null; + lastErrorAt: number | null; + lastError: string | null; +} + +export interface BridgeStatusReader { + snapshot(): BridgeStatusSnapshot; +} + +export interface BridgeStatusWriter extends BridgeStatusReader { + markConnected(): void; + markDisconnected(message: string): void; + startHeartbeat(): { stop(): void }; +} + +interface ReaderOptions { + isHermesChatRunning: () => boolean; + now?: () => number; + staleMs?: number; +} + +interface WriterOptions { + now?: () => number; + heartbeatMs?: number; +} + +function readStatus(statusFile: string): BridgeStatusSnapshot | null { + try { + const parsed = JSON.parse( + fs.readFileSync(statusFile, "utf8"), + ) as Partial; + if ( + parsed.status === "connected" || + parsed.status === "reconnecting" || + parsed.status === "disconnected" || + parsed.status === "unknown" + ) { + return { + status: parsed.status, + lastOkAt: + typeof parsed.lastOkAt === "number" ? parsed.lastOkAt : null, + lastErrorAt: + typeof parsed.lastErrorAt === "number" ? parsed.lastErrorAt : null, + lastError: + typeof parsed.lastError === "string" ? parsed.lastError : null, + }; + } + } catch { + // Missing and corrupt status files both mean there is no live writer. + } + return null; +} + +function errorAt( + status: BridgeStatusSnapshot | null, + observedAt: number, +): number { + return status?.lastErrorAt ?? status?.lastOkAt ?? observedAt; +} + +/** + * Read-only view used by the standalone Hermes Viewer daemon. + * + * The nested health `bridge` describes the Python provider ↔ Node stdio + * transport. The Viewer daemon is a separate HTTP process, so it must never + * refresh this file or claim that Hermes chat is connected on its own behalf. + */ +export function createBridgeStatusReader( + statusFile: string, + options: ReaderOptions, +): BridgeStatusReader { + const now = options.now ?? Date.now; + const staleMs = options.staleMs ?? BRIDGE_STATUS_STALE_MS; + const firstObservedAt = now(); + + return { + snapshot() { + const status = readStatus(statusFile); + const observedAt = now(); + const freshConnected = + status?.status === "connected" && + status.lastOkAt != null && + observedAt - status.lastOkAt <= staleMs; + + // A fresh heartbeat is stronger evidence than process-name probing, + // which can miss valid Hermes command-line shapes on some platforms. + if (freshConnected) return { ...status }; + + const chatRunning = options.isHermesChatRunning(); + if (chatRunning) { + const heartbeatStale = + status?.status === "connected" && status.lastOkAt != null; + return { + status: "reconnecting", + lastOkAt: status?.lastOkAt ?? null, + lastErrorAt: errorAt(status, firstObservedAt), + lastError: heartbeatStale + ? "Hermes bridge heartbeat is stale" + : "Hermes chat is running; waiting for memory bridge", + }; + } + + if (status?.status === "disconnected") return { ...status }; + + return { + status: "disconnected", + lastOkAt: status?.lastOkAt ?? null, + lastErrorAt: errorAt(status, firstObservedAt), + lastError: "Hermes chat is not connected", + }; + }, + }; +} + +/** The stdio bridge is the sole writer of Hermes transport status. */ +export function createBridgeStatusWriter( + statusFile: string, + options: WriterOptions = {}, +): BridgeStatusWriter { + const now = options.now ?? Date.now; + const heartbeatMs = options.heartbeatMs ?? BRIDGE_STATUS_HEARTBEAT_MS; + let status: BridgeStatusSnapshot = { + status: "unknown", + lastOkAt: null, + lastErrorAt: null, + lastError: null, + }; + + function writeStatus(next: BridgeStatusSnapshot): void { + status = next; + try { + fs.mkdirSync(path.dirname(statusFile), { recursive: true }); + fs.writeFileSync(statusFile, JSON.stringify(next), "utf8"); + } catch { + // Status display must never affect chat capture. + } + } + + function markConnected(): void { + writeStatus({ + status: "connected", + lastOkAt: now(), + lastErrorAt: status.lastErrorAt, + lastError: status.lastError, + }); + } + + return { + snapshot() { + return { ...(readStatus(statusFile) ?? status) }; + }, + markConnected, + markDisconnected(message: string) { + writeStatus({ + status: "disconnected", + lastOkAt: status.lastOkAt, + lastErrorAt: now(), + lastError: message, + }); + }, + startHeartbeat() { + const timer = setInterval(markConnected, heartbeatMs); + timer.unref?.(); + return { + stop() { + clearInterval(timer); + }, + }; + }, + }; +} diff --git a/apps/memos-local-plugin/tests/unit/bridge/status.test.ts b/apps/memos-local-plugin/tests/unit/bridge/status.test.ts new file mode 100644 index 000000000..76aab85e9 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/bridge/status.test.ts @@ -0,0 +1,203 @@ +import { + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + createBridgeStatusReader, + createBridgeStatusWriter, + type BridgeStatusSnapshot, +} from "../../../bridge/status.js"; + +const STALE_MS = 20_000; + +describe("Hermes bridge status ownership", () => { + const tempDirs: string[] = []; + + afterEach(() => { + vi.useRealTimers(); + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } + }); + + function statusFile(): string { + const dir = mkdtempSync(join(tmpdir(), "memos-bridge-status-")); + tempDirs.push(dir); + return join(dir, "bridge-status.json"); + } + + function writeStatus(file: string, status: BridgeStatusSnapshot): void { + writeFileSync(file, JSON.stringify(status), "utf8"); + } + + it("keeps a fresh stdio heartbeat connected even if process detection misses Hermes", () => { + const file = statusFile(); + writeStatus(file, { + status: "connected", + lastOkAt: 90_000, + lastErrorAt: null, + lastError: null, + }); + + const isHermesChatRunning = vi.fn(() => false); + const reader = createBridgeStatusReader(file, { + isHermesChatRunning, + now: () => 100_000, + staleMs: STALE_MS, + }); + + expect(reader.snapshot()).toEqual({ + status: "connected", + lastOkAt: 90_000, + lastErrorAt: null, + lastError: null, + }); + expect(isHermesChatRunning).not.toHaveBeenCalled(); + }); + + it("reports daemon-only startup as not connected without creating a file", () => { + const file = statusFile(); + const reader = createBridgeStatusReader(file, { + isHermesChatRunning: () => false, + now: () => 100_000, + staleMs: STALE_MS, + }); + + expect(reader.snapshot()).toEqual({ + status: "disconnected", + lastOkAt: null, + lastErrorAt: 100_000, + lastError: "Hermes chat is not connected", + }); + expect(() => statSync(file)).toThrow(); + }); + + it("normalizes a stale heartbeat to not connected when Hermes is not running", () => { + const file = statusFile(); + const stale = { + status: "connected" as const, + lastOkAt: 1_000, + lastErrorAt: null, + lastError: null, + }; + writeStatus(file, stale); + const before = statSync(file).mtimeNs; + + const reader = createBridgeStatusReader(file, { + isHermesChatRunning: () => false, + now: () => 100_000, + staleMs: STALE_MS, + }); + + expect(reader.snapshot()).toEqual({ + status: "disconnected", + lastOkAt: 1_000, + lastErrorAt: 1_000, + lastError: "Hermes chat is not connected", + }); + expect(JSON.parse(readFileSync(file, "utf8"))).toEqual(stale); + expect(statSync(file).mtimeNs).toBe(before); + }); + + it("reports reconnecting when Hermes is running but its heartbeat is stale", () => { + const file = statusFile(); + writeStatus(file, { + status: "connected", + lastOkAt: 1_000, + lastErrorAt: null, + lastError: null, + }); + + const reader = createBridgeStatusReader(file, { + isHermesChatRunning: () => true, + now: () => 100_000, + staleMs: STALE_MS, + }); + + expect(reader.snapshot()).toEqual({ + status: "reconnecting", + lastOkAt: 1_000, + lastErrorAt: 1_000, + lastError: "Hermes bridge heartbeat is stale", + }); + }); + + it("reports a stable waiting state without creating a file when Hermes is starting", () => { + const file = statusFile(); + const reader = createBridgeStatusReader(file, { + isHermesChatRunning: () => true, + now: () => 100_000, + staleMs: STALE_MS, + }); + + const expected = { + status: "reconnecting" as const, + lastOkAt: null, + lastErrorAt: 100_000, + lastError: "Hermes chat is running; waiting for memory bridge", + }; + expect(reader.snapshot()).toEqual(expected); + expect(reader.snapshot()).toEqual(expected); + expect(() => statSync(file)).toThrow(); + }); + + it("does not overwrite an explicit stdio disconnect while reporting reconnecting", () => { + const file = statusFile(); + const disconnected = { + status: "disconnected" as const, + lastOkAt: 90_000, + lastErrorAt: 95_000, + lastError: "Hermes chat disconnected", + }; + writeStatus(file, disconnected); + const before = statSync(file).mtimeNs; + const reader = createBridgeStatusReader(file, { + isHermesChatRunning: () => true, + now: () => 100_000, + staleMs: STALE_MS, + }); + + expect(reader.snapshot()).toEqual({ + status: "reconnecting", + lastOkAt: 90_000, + lastErrorAt: 95_000, + lastError: "Hermes chat is running; waiting for memory bridge", + }); + expect(JSON.parse(readFileSync(file, "utf8"))).toEqual(disconnected); + expect(statSync(file).mtimeNs).toBe(before); + }); + + it("advances only the stdio writer heartbeat and stops cleanly", () => { + vi.useFakeTimers(); + vi.setSystemTime(1_000); + const file = statusFile(); + const writer = createBridgeStatusWriter(file, { heartbeatMs: 5_000 }); + + writer.markConnected(); + expect(JSON.parse(readFileSync(file, "utf8")).lastOkAt).toBe(1_000); + + const heartbeat = writer.startHeartbeat(); + vi.advanceTimersByTime(5_000); + expect(JSON.parse(readFileSync(file, "utf8")).lastOkAt).toBe(6_000); + + heartbeat.stop(); + vi.advanceTimersByTime(10_000); + expect(JSON.parse(readFileSync(file, "utf8")).lastOkAt).toBe(6_000); + + writer.markDisconnected("Hermes chat disconnected"); + expect(JSON.parse(readFileSync(file, "utf8"))).toMatchObject({ + status: "disconnected", + lastOkAt: 6_000, + lastErrorAt: 16_000, + lastError: "Hermes chat disconnected", + }); + }); +}); From d6ab77f03b7d75880c9bc58ca283bea0fe386d79 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Wed, 26 Aug 2026 10:05:02 +0000 Subject: [PATCH 32/34] docs(storage): document migrator collision-heal allowlist per review --- apps/memos-local-plugin/core/storage/migrator.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/memos-local-plugin/core/storage/migrator.ts b/apps/memos-local-plugin/core/storage/migrator.ts index 270a2532c..f91d1ca62 100644 --- a/apps/memos-local-plugin/core/storage/migrator.ts +++ b/apps/memos-local-plugin/core/storage/migrator.ts @@ -215,6 +215,9 @@ function applyMigration(db: StorageDb, file: MigrationFile): void { } return; } + // Keep REPAIRABLE_UNDER_NAME_COLLISION (bottom of file) in sync when + // adding guarded cases here — a guarded case missing from that set + // silently loses repair-under-name-collision (the file is just skipped). if (file.version === 13 && file.name === "traces-ts-index") { // Same guard as 012: some test harnesses build partial schemas without a // `traces` table; the index is meaningless there and must not fail boot. @@ -450,7 +453,14 @@ function getAppliedMigrationNames(db: StorageDb): Map { * the conservative behaviour: a version-number collision skips the file. */ const REPAIRABLE_UNDER_NAME_COLLISION = new Set([ - 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, + 3, 4, 5, 6, 7, 8, 9, 10, + // 11 (hub-sharing) intentionally omitted: it has no guarded apply path in + // applyMigration(), and hub runtime tables are opt-in team-sharing state a + // collision heal must never create implicitly. Its SQL is idempotent (all + // CREATE ... IF NOT EXISTS), but policy keeps the conservative skip; pinned + // by the "keeps skipping a version recorded under a foreign name when that + // migration is not repairable" test. Give 011 a guarded path before adding. + 12, 13, ]); /** From b90b5c910cb41be85231e8980eb795c11a653d6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Wed, 26 Aug 2026 19:39:17 +0800 Subject: [PATCH 33/34] fix(plugin): harden newest-trace index rollout --- .../core/pipeline/memory-core.ts | 11 ++-- ...s-ts-index.sql => 018-traces-ts-index.sql} | 0 .../core/storage/migrator.ts | 64 +++---------------- .../core/storage/repos/traces.ts | 7 ++ .../tests/unit/pipeline/memory-core.test.ts | 16 +++++ .../tests/unit/storage/migrator.test.ts | 56 +++++++++++----- .../tests/unit/storage/repos.test.ts | 1 + 7 files changed, 76 insertions(+), 79 deletions(-) rename apps/memos-local-plugin/core/storage/migrations/{013-traces-ts-index.sql => 018-traces-ts-index.sql} (100%) diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index bd64d5e08..660ea737c 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -2051,8 +2051,9 @@ export function createMemoryCore( // actually been able to talk to the configured upstream. See #1596. const effectiveConfig = diskConfig ?? handle.config; - const llmInfo = llmHealth(handle.llm, latestTraceTs()); - const embedderInfo = embedderHealth(handle.embedder, latestTraceTs()); + const latestTraceTimestamp = latestTraceTs(); + const llmInfo = llmHealth(handle.llm, latestTraceTimestamp); + const embedderInfo = embedderHealth(handle.embedder, latestTraceTimestamp); applyConfiguredModelDisplay(effectiveConfig, llmInfo, embedderInfo); const skillEvolverInfo = resolveSkillEvolver( @@ -2065,7 +2066,7 @@ export function createMemoryCore( // in that case anyway. handle.reflectLlm ?? handle.llm, llmInfo, - latestTraceTs(), + latestTraceTimestamp, ); // NOTE: we deliberately do NOT fall back to `api_logs`-stored @@ -2106,9 +2107,7 @@ export function createMemoryCore( function latestTraceTs(): number | null { try { - const rows = handle.repos.traces.list({ limit: 1 }); - if (rows.length === 0) return null; - return rows[0]?.ts ?? null; + return handle.repos.traces.latestTimestamp(); } catch { return null; } diff --git a/apps/memos-local-plugin/core/storage/migrations/013-traces-ts-index.sql b/apps/memos-local-plugin/core/storage/migrations/018-traces-ts-index.sql similarity index 100% rename from apps/memos-local-plugin/core/storage/migrations/013-traces-ts-index.sql rename to apps/memos-local-plugin/core/storage/migrations/018-traces-ts-index.sql diff --git a/apps/memos-local-plugin/core/storage/migrator.ts b/apps/memos-local-plugin/core/storage/migrator.ts index f91d1ca62..b858f7aa3 100644 --- a/apps/memos-local-plugin/core/storage/migrator.ts +++ b/apps/memos-local-plugin/core/storage/migrator.ts @@ -90,7 +90,7 @@ function assertMonotonic(files: MigrationFile[]): void { export function runMigrations(db: StorageDb, dir: string = defaultMigrationsDir()): MigrationsResult { ensureSchemaMigrationsTable(db); const allFiles = discoverMigrations(dir); - const appliedNames = getAppliedMigrationNames(db); + const appliedVersions = getAppliedVersions(db); const applied: MigrationsResult["applied"] = []; let skipped = 0; @@ -102,35 +102,22 @@ export function runMigrations(db: StorageDb, dir: string = defaultMigrationsDir( // input, so turning unsafe mode on for the migration phase is safe. // `.unsafeMode()` may not be toggled inside a transaction, so we flip it // at the outer boundary. - const isPending = (file: MigrationFile): boolean => { - const recordedName = appliedNames.get(file.version); - // Pending when never recorded, or recorded under a FOREIGN name (release- - // train collision) for a migration whose apply path is safe to re-run. - if (recordedName === undefined) return true; - return recordedName !== file.name && REPAIRABLE_UNDER_NAME_COLLISION.has(file.version); - }; const needsUnsafe = allFiles.some( - (f) => isPending(f) && migrationNeedsUnsafeMode(f.fullPath), + (f) => !appliedVersions.has(f.version) && migrationNeedsUnsafeMode(f.fullPath), ); if (needsUnsafe) db.raw.unsafeMode(true); try { for (const file of allFiles) { - if (!isPending(file)) { + if (appliedVersions.has(file.version)) { skipped++; continue; } const t0 = now(); db.tx(() => { applyMigration(db, file); - // Upsert: when the version row existed under a foreign name (release- - // train collision), repair the bookkeeping to reflect what THIS train - // last ensured. When the row is fresh, this behaves like the plain - // INSERT it replaces. db.prepare( - `INSERT INTO schema_migrations (version, name, applied_at) - VALUES (@version, @name, @applied_at) - ON CONFLICT(version) DO UPDATE SET name = excluded.name, applied_at = excluded.applied_at`, + `INSERT INTO schema_migrations (version, name, applied_at) VALUES (@version, @name, @applied_at)`, ).run({ version: file.version, name: file.name, applied_at: now() }); }); const durationMs = now() - t0; @@ -215,10 +202,7 @@ function applyMigration(db: StorageDb, file: MigrationFile): void { } return; } - // Keep REPAIRABLE_UNDER_NAME_COLLISION (bottom of file) in sync when - // adding guarded cases here — a guarded case missing from that set - // silently loses repair-under-name-collision (the file is just skipped). - if (file.version === 13 && file.name === "traces-ts-index") { + if (file.version === 18 && file.name === "traces-ts-index") { // Same guard as 012: some test harnesses build partial schemas without a // `traces` table; the index is meaningless there and must not fail boot. if (tableExists(db, "traces")) { @@ -424,45 +408,13 @@ function ensureSchemaMigrationsTable(db: StorageDb): void { ); } -/** - * Applied migrations keyed by version number, valued by name. - * - * The npm plugin train and the monorepo train have historically reused - * version numbers for DIFFERENT migrations (e.g. a database migrated by the - * other train carries `(13, '')`). Version-only bookkeeping - * makes such databases silently skip any same-numbered migration shipped by - * this build -- including additive repair migrations that are safe to apply. - */ -function getAppliedMigrationNames(db: StorageDb): Map { +function getAppliedVersions(db: StorageDb): Set { const rows = db - .prepare( - `SELECT version, name FROM schema_migrations`, - ) + .prepare(`SELECT version FROM schema_migrations`) .all(); - return new Map(rows.map((r) => [r.version, r.name])); + return new Set(rows.map((r) => r.version)); } -/** - * Migrations whose `applyMigration()` path is safe to execute even when the - * same version number was already recorded under a DIFFERENT name (see - * `getAppliedMigrationNames`). Every entry here is guarded: it either no-ops - * through existence checks (`ensureColumn`, `tableExists`, `CREATE ... IF NOT - * EXISTS`) or performs a strictly additive change, so running it against a - * database that already has the equivalent objects from another train is a - * cheap no-op rather than a corruption risk. Anything NOT listed here keeps - * the conservative behaviour: a version-number collision skips the file. - */ -const REPAIRABLE_UNDER_NAME_COLLISION = new Set([ - 3, 4, 5, 6, 7, 8, 9, 10, - // 11 (hub-sharing) intentionally omitted: it has no guarded apply path in - // applyMigration(), and hub runtime tables are opt-in team-sharing state a - // collision heal must never create implicitly. Its SQL is idempotent (all - // CREATE ... IF NOT EXISTS), but policy keeps the conservative skip; pinned - // by the "keeps skipping a version recorded under a foreign name when that - // migration is not repairable" test. Give 011 a guarded path before adding. - 12, 13, -]); - /** * Convenience helper for tests / CLIs: open, migrate, return. */ diff --git a/apps/memos-local-plugin/core/storage/repos/traces.ts b/apps/memos-local-plugin/core/storage/repos/traces.ts index d78094872..d8bcea5e3 100644 --- a/apps/memos-local-plugin/core/storage/repos/traces.ts +++ b/apps/memos-local-plugin/core/storage/repos/traces.ts @@ -106,6 +106,9 @@ export function makeTracesRepo(db: StorageDb) { const selectById = db.prepare<{ id: string }, RawTraceRow>( `SELECT ${COLUMNS.join(", ")} FROM traces WHERE id=@id`, ); + const selectLatestTimestamp = db.prepare( + `SELECT ts FROM traces ORDER BY ts DESC, id DESC LIMIT 1`, + ); return { insert(row: TraceRow): void { @@ -135,6 +138,10 @@ export function makeTracesRepo(db: StorageDb) { return mapRow(r); }, + latestTimestamp(): number | null { + return selectLatestTimestamp.get()?.ts ?? null; + }, + getManyByIds(ids: readonly TraceId[]): TraceRow[] { if (ids.length === 0) return []; const placeholders = buildInClause(ids.length); diff --git a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts index c3067c1de..2759ae5bd 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts @@ -171,6 +171,22 @@ describe("MemoryCore façade", () => { expect(h.llm.available).toBe(false); }); + it("reads the latest trace timestamp only once per health snapshot", async () => { + const latestTimestamp = vi.spyOn(db!.repos.traces, "latestTimestamp"); + pipeline = createPipeline(buildDeps(db!)); + core = createMemoryCore( + pipeline, + resolveHome("openclaw", "/tmp/memos-mc-test"), + "test-1.0.0", + ); + await core.init(); + latestTimestamp.mockClear(); + + await core.health(); + + expect(latestTimestamp).toHaveBeenCalledTimes(1); + }); + it("reloads the hub runtime when hub config changes without a process restart", async () => { const home = await makeTmpHome({ agent: "openclaw", diff --git a/apps/memos-local-plugin/tests/unit/storage/migrator.test.ts b/apps/memos-local-plugin/tests/unit/storage/migrator.test.ts index 9e23e810b..933f59977 100644 --- a/apps/memos-local-plugin/tests/unit/storage/migrator.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/migrator.test.ts @@ -206,7 +206,7 @@ describe("storage/migrator", () => { } }); - it("013-traces-ts-index creates the bare-ts index and repairs train-collision bookkeeping", () => { + it("018-traces-ts-index creates the bare-ts index without rewriting historical migrations", () => { // Regression test for the Aug 2026 restart storm: `latestTraceTs()` runs // an unfiltered newest-first trace read several times per /api/v1/health // request. No existing index leads with bare `ts`, so every call was a @@ -214,17 +214,14 @@ describe("storage/migrator", () => { // synchronous better-sqlite3 event loop long enough that health probes // timed out and a liveness watchdog restart-looped the daemon forever. // - // This test also exercises the release-train heal: databases migrated by - // the other train carry schema_migrations rows whose VERSION numbers - // collide with different NAMES (observed: rows 13/14 named - // skill-repair-origin / episode-outcome). Repairable additive migrations - // must still apply and repair their bookkeeping row. + // Published and development release trains have already used versions + // 13-17 for unrelated migrations. The new index must therefore use 018 + // and preserve those historical bookkeeping rows verbatim. const { dbPath, cleanup } = tmpDb(); cleanups.push(cleanup); const db = openDb({ filepath: dbPath, agent: "openclaw" }); try { - // Simulate a database previously migrated by the OTHER release train: - // versions 13/14 exist under foreign names before this build runs. + // Simulate a database previously migrated by the other release trains. db.exec(` CREATE TABLE schema_migrations ( version INTEGER PRIMARY KEY, @@ -233,11 +230,19 @@ describe("storage/migrator", () => { ) STRICT; `); db.exec( - `INSERT INTO schema_migrations (version, name, applied_at) VALUES (13, 'skill-repair-origin', 1), (14, 'episode-outcome', 1)`, + `INSERT INTO schema_migrations (version, name, applied_at) VALUES + (13, 'skill-repair-origin', 1), + (14, 'episode-outcome', 2), + (15, 'policy-merge-family', 3), + (16, 'episode-policy-injections', 4), + (17, 'evolution-jobs', 5)`, ); const result = runMigrations(db); - expect(result.applied.map((m) => m.name)).toContain("traces-ts-index"); + expect(result.applied).toContainEqual(expect.objectContaining({ + version: 18, + name: "traces-ts-index", + })); // The index exists... const index = db @@ -247,20 +252,37 @@ describe("storage/migrator", () => { .get(); expect(index?.name).toBe("idx_traces_ts"); - // ...the plan for newest-first reads uses it instead of a table scan, - // and the foreign bookkeeping row was repaired to THIS train's name. + // ...and the plan for newest-first reads uses it instead of a table + // scan plus a temporary sort. const detail = db .prepare( `SELECT sql FROM sqlite_master WHERE type='index' AND name='idx_traces_ts'`, ) .get(); expect(detail?.sql).toContain("ts DESC"); - const repaired = db - .prepare( - `SELECT name FROM schema_migrations WHERE version = 13`, + const plan = db + .prepare( + `EXPLAIN QUERY PLAN SELECT ts FROM traces ORDER BY ts DESC, id DESC LIMIT 1`, ) - .get(); - expect(repaired?.name).toBe("traces-ts-index"); + .all() + .map((row) => row.detail) + .join("\n"); + expect(plan).toContain("USING COVERING INDEX idx_traces_ts"); + expect(plan).not.toContain("USE TEMP B-TREE"); + + const historicalRows = db + .prepare( + `SELECT version, name, applied_at FROM schema_migrations + WHERE version BETWEEN 13 AND 17 ORDER BY version`, + ) + .all(); + expect(historicalRows).toEqual([ + { version: 13, name: "skill-repair-origin", applied_at: 1 }, + { version: 14, name: "episode-outcome", applied_at: 2 }, + { version: 15, name: "policy-merge-family", applied_at: 3 }, + { version: 16, name: "episode-policy-injections", applied_at: 4 }, + { version: 17, name: "evolution-jobs", applied_at: 5 }, + ]); // Re-running is idempotent: everything counts as skipped. const again = runMigrations(db); diff --git a/apps/memos-local-plugin/tests/unit/storage/repos.test.ts b/apps/memos-local-plugin/tests/unit/storage/repos.test.ts index 4da3d77b8..ea1904901 100644 --- a/apps/memos-local-plugin/tests/unit/storage/repos.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/repos.test.ts @@ -144,6 +144,7 @@ describe("storage/repos — happy paths", () => { const all = repos.traces.list({ sessionId: "s" }); expect(all.length).toBe(3); expect(all[0]!.ts).toBeGreaterThan(all[1]!.ts); // newest first by default + expect(repos.traces.latestTimestamp()).toBe(30); const highAbs = repos.traces.list({ minAbsValue: 0.8 }); expect(highAbs.map((t) => t.id)).toEqual(["t0"]); From d76c78aad0dcc805e7b1d045bad80e1a1661e6b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=81=E5=9C=A8=E5=90=B5=E7=9D=80=E5=90=83=E7=B3=96?= Date: Wed, 26 Aug 2026 20:27:14 +0800 Subject: [PATCH 34/34] fix(plugin): resolve auxiliary reasoning capabilities --- .../adapters/deepseek-harness/host-llm.ts | 111 ++++++++- .../adapters/deepseek-harness/index.ts | 17 ++ .../deepseek-harness-host-llm.test.ts | 226 +++++++++++++++++- .../adapters/deepseek-harness-runtime.test.ts | 32 ++- 4 files changed, 370 insertions(+), 16 deletions(-) diff --git a/apps/memos-local-plugin/adapters/deepseek-harness/host-llm.ts b/apps/memos-local-plugin/adapters/deepseek-harness/host-llm.ts index 17618d2c0..a755efb98 100644 --- a/apps/memos-local-plugin/adapters/deepseek-harness/host-llm.ts +++ b/apps/memos-local-plugin/adapters/deepseek-harness/host-llm.ts @@ -10,6 +10,7 @@ import { type GenerateOptions, type LlmCallConfig, type LlmFailure, + type LlmResolvedModelInfo, type PreparedLlmCall, type StreamChunk, } from "@deepseek-ai/dsh-llm"; @@ -29,6 +30,7 @@ const HOST_LLM_TIMEOUT_CODE = "MEMOS_DSH_HOST_LLM_TIMEOUT"; const HOST_LLM_MESSAGE_SOURCE = "memos-local-memory"; const NO_REASONING_EFFORT = ReasoningEffortId("off"); const UNSUPPORTED_REASONING_EFFORT = "UNSUPPORTED_REASONING_EFFORT"; +const MODEL_CAPABILITY_TTL_MS = 10 * 60 * 1_000; /** Atomic provider/model route captured from the DSH agent that owns a turn. */ export interface DeepSeekHarnessLlmRoute { @@ -40,12 +42,34 @@ export interface DeepSeekHarnessLlmRoute { /** Public subset of DSH's LLM runtime used by this adapter. */ export interface DeepSeekHarnessLlmLike { + resolveModelInfo( + provider: string, + model: string, + signal?: AbortSignal, + ): Promise; prepareCall( config: LlmCallConfig, signal?: AbortSignal, ): Promise; } +export interface DeepSeekHarnessHostLlmBridge extends HostLlmBridge { + /** Drop exact-route capability snapshots after a DSH adapter topology update. */ + invalidateModelCapabilities(): void; +} + +type AuxiliaryReasoningCapability = "off" | "plain"; + +interface CapabilityCacheEntry { + readonly expiresAt: number; + readonly value: Promise; +} + +interface CapabilityCache { + readonly entries: Map; + generation: number; +} + /** * Async route scope for MemOS work spawned by one DSH session. * @@ -95,9 +119,17 @@ export interface CreateDeepSeekHarnessHostLlmBridgeOptions { */ export function createDeepSeekHarnessHostLlmBridge( options: CreateDeepSeekHarnessHostLlmBridgeOptions, -): HostLlmBridge { +): DeepSeekHarnessHostLlmBridge { + const capabilityCache: CapabilityCache = { + entries: new Map(), + generation: 0, + }; return { id: HOST_LLM_BRIDGE_ID, + invalidateModelCapabilities(): void { + capabilityCache.generation++; + capabilityCache.entries.clear(); + }, async complete(input: HostLlmCompleteInput): Promise { const route = options.routes.current(); if (!route) { @@ -122,6 +154,7 @@ export function createDeepSeekHarnessHostLlmBridge( input, route, callDeadline.signal, + capabilityCache, ); const request = createGenerateOptions(input, route, prepared.config); request.signal = callDeadline.signal; @@ -217,20 +250,30 @@ function createGenerateOptions( * * Retrieval filters and JSON extractors intentionally use small output caps. * Reusing a conversation's high reasoning effort can spend that entire cap on - * reasoning and produce no JSON/text. DSH effort ids are adapter-owned, so the - * exact registered adapter validates the branded conventional `off` id. Only - * an explicit unsupported-effort result retries without it, preserving the - * adapter/provider default. prepareCall performs no provider generation I/O - * and binds that validation to the same registration used for dispatch, even - * if HMR replaces the route before the returned stream starts. + * reasoning and produce no JSON/text. Resolve exact-model metadata through + * DSH, then cache whether its adapter advertises the conventional `off` id. + * prepareCall remains the final registration-bound authority: if HMR changes + * the route after metadata resolution, an explicit unsupported-effort result + * refreshes the cache and retries with the adapter/provider default. */ async function prepareAuxiliaryCall( llm: DeepSeekHarnessLlmLike, input: HostLlmCompleteInput, route: DeepSeekHarnessLlmRoute, signal: AbortSignal, + capabilityCache: CapabilityCache, ): Promise { const config = createCallConfig(input, route); + const capability = await resolveAuxiliaryReasoningCapability( + llm, + route, + signal, + capabilityCache, + ); + if (capability === "plain") { + return llm.prepareCall(config, signal); + } + const preparationGeneration = capabilityCache.generation; try { return await llm.prepareCall( { ...config, reasoningEffort: NO_REASONING_EFFORT }, @@ -243,10 +286,64 @@ async function prepareAuxiliaryCall( ) { throw error; } + // The exact adapter may have changed after the metadata lookup. Preserve + // prepareCall's registration-bound validation as the final authority and + // remember the corrected capability only if no newer topology update has + // already invalidated this preparation generation. + if (capabilityCache.generation === preparationGeneration) { + rememberAuxiliaryReasoningCapability(route, "plain", capabilityCache); + } return llm.prepareCall(config, signal); } } +function capabilityRouteKey(route: DeepSeekHarnessLlmRoute): string { + return JSON.stringify([route.provider, route.model]); +} + +function rememberAuxiliaryReasoningCapability( + route: DeepSeekHarnessLlmRoute, + capability: AuxiliaryReasoningCapability, + cache: CapabilityCache, +): void { + cache.entries.set(capabilityRouteKey(route), { + expiresAt: Date.now() + MODEL_CAPABILITY_TTL_MS, + value: Promise.resolve(capability), + }); +} + +async function resolveAuxiliaryReasoningCapability( + llm: DeepSeekHarnessLlmLike, + route: DeepSeekHarnessLlmRoute, + signal: AbortSignal, + cache: CapabilityCache, +): Promise { + const key = capabilityRouteKey(route); + const cached = cache.entries.get(key); + if (cached && cached.expiresAt > Date.now()) { + return cached.value; + } + if (cached) cache.entries.delete(key); + + let entry: CapabilityCacheEntry; + const value = llm.resolveModelInfo(route.provider, route.model, signal) + .then((info): AuxiliaryReasoningCapability => ( + info.reasoning?.efforts.some((effort) => effort.id === NO_REASONING_EFFORT) + ? "off" + : "plain" + )) + .catch((error: unknown) => { + if (cache.entries.get(key) === entry) cache.entries.delete(key); + throw error; + }); + entry = { + expiresAt: Date.now() + MODEL_CAPABILITY_TTL_MS, + value, + }; + cache.entries.set(key, entry); + return value; +} + function createCallConfig( input: HostLlmCompleteInput, route: DeepSeekHarnessLlmRoute, diff --git a/apps/memos-local-plugin/adapters/deepseek-harness/index.ts b/apps/memos-local-plugin/adapters/deepseek-harness/index.ts index 5c839f104..bc63ba5d6 100644 --- a/apps/memos-local-plugin/adapters/deepseek-harness/index.ts +++ b/apps/memos-local-plugin/adapters/deepseek-harness/index.ts @@ -36,6 +36,7 @@ import { import { createDeepSeekHarnessHostLlmBridge, DeepSeekHarnessLlmRouteContext, + type DeepSeekHarnessHostLlmBridge, } from "./host-llm.js"; import { registerDeepSeekHarnessTools } from "./tools.js"; @@ -126,6 +127,16 @@ export function configureDeepSeekHarnessHostLlm( }); } +/** Refresh exact-model capability snapshots whenever DSH replaces an adapter. */ +export function registerDeepSeekHarnessHostLlmCapabilityInvalidation( + ctx: Context, + bridge: DeepSeekHarnessHostLlmBridge, +): () => void { + return ctx.on("llm/adapters-updated", () => { + bridge.invalidateModelCapabilities(); + }); +} + /** Autonomous recovery has no owning DSH turn from which to capture a route. */ export function deepSeekHarnessAutoRecoveryEnabled(config: ResolvedConfig): boolean { return config.llm.provider.trim().toLowerCase() !== "host" || @@ -247,6 +258,12 @@ export async function apply( const hostLlmBridge = config.hostLlmEnabled ? createDeepSeekHarnessHostLlmBridge({ llm: ctx.llm, routes }) : null; + if (hostLlmBridge) { + registrations.push(registerDeepSeekHarnessHostLlmCapabilityInvalidation( + ctx, + hostLlmBridge, + )); + } const autoRecoveryEnabled = deepSeekHarnessAutoRecoveryEnabled(memoryConfig); core = await bootstrapMemoryCore({ diff --git a/apps/memos-local-plugin/tests/unit/adapters/deepseek-harness-host-llm.test.ts b/apps/memos-local-plugin/tests/unit/adapters/deepseek-harness-host-llm.test.ts index 87b826afc..493fba9e7 100644 --- a/apps/memos-local-plugin/tests/unit/adapters/deepseek-harness-host-llm.test.ts +++ b/apps/memos-local-plugin/tests/unit/adapters/deepseek-harness-host-llm.test.ts @@ -34,8 +34,27 @@ function streamFrom( config: LlmCallConfig, signal?: AbortSignal, ) => void, + observeResolution?: (provider: string, model: string) => void, ): DeepSeekHarnessLlmLike { return { + async resolveModelInfo(provider, model) { + observeResolution?.(provider, model); + return { + provider, + id: model, + name: model, + ...(reasoningEfforts.length === 0 + ? {} + : { + reasoning: { + efforts: reasoningEfforts.map((effort) => ({ + id: ReasoningEffortId(effort), + name: effort, + })), + }, + }), + }; + }, async prepareCall(config, signal) { observePreparation?.(config, signal); if ( @@ -192,6 +211,7 @@ describe("DeepSeek Harness host LLM bridge", () => { it("does not invent an off effort when the exact model does not advertise it", async () => { let request: GenerateOptions | undefined; const preparations: LlmCallConfig[] = []; + let resolutions = 0; const routes = new DeepSeekHarnessLlmRouteContext(); const bridge = createDeepSeekHarnessHostLlmBridge({ llm: streamFrom([ @@ -201,14 +221,18 @@ describe("DeepSeek Harness host LLM bridge", () => { request = options; }, [], (config) => { preparations.push(config); + }, () => { + resolutions++; }), routes, }); - const result = await routes.run( + const complete = () => routes.run( { provider: "openai", model: "gpt-test", reasoningEffort: "high" }, () => bridge.complete({ messages: [{ role: "user", content: "hello" }] }), ); + const result = await complete(); + await complete(); expect(request).not.toHaveProperty("system"); expect(request).not.toHaveProperty("reasoningEffort"); @@ -216,24 +240,197 @@ describe("DeepSeek Harness host LLM bridge", () => { expect(request).not.toHaveProperty("temperature"); expect(request).not.toHaveProperty("maxTokens"); expect(preparations).toEqual([ - { - provider: "openai", - model: "gpt-test", - reasoningEffort: ReasoningEffortId("off"), - }, + { provider: "openai", model: "gpt-test" }, { provider: "openai", model: "gpt-test" }, ]); + expect(resolutions).toBe(1); expect(result).not.toHaveProperty("usage"); expect(result.text).toBe("ok"); }); + it("invalidates resolved effort capabilities when the DSH adapter changes", async () => { + let supportsOff = false; + let resolutions = 0; + const preparations: string[] = []; + const routes = new DeepSeekHarnessLlmRouteContext(); + const bridge = createDeepSeekHarnessHostLlmBridge({ + llm: { + async resolveModelInfo(provider, model) { + resolutions++; + return { + provider, + id: model, + name: model, + ...(supportsOff + ? { + reasoning: { + efforts: [{ id: ReasoningEffortId("off"), name: "Off" }], + }, + } + : {}), + }; + }, + async prepareCall(config) { + preparations.push(config.reasoningEffort ?? "plain"); + if (config.reasoningEffort !== undefined && !supportsOff) { + throw new LlmError("unsupported", "UNSUPPORTED_REASONING_EFFORT"); + } + return preparedCall(config, () => (async function* () { + yield { type: "text-delta", index: 0, text: "ok" } as StreamChunk; + yield { type: "finish", reason: { kind: "stop" } } as StreamChunk; + })()); + }, + }, + routes, + }); + const complete = () => routes.run( + { provider: "openai", model: "gpt-test" }, + () => bridge.complete({ messages: [{ role: "user", content: "hello" }] }), + ); + + await complete(); + supportsOff = true; + await complete(); + bridge.invalidateModelCapabilities(); + await complete(); + + expect(preparations).toEqual(["plain", "plain", "off"]); + expect(resolutions).toBe(2); + }); + + it("coalesces concurrent capability lookups for one route", async () => { + let resolutions = 0; + const routes = new DeepSeekHarnessLlmRouteContext(); + const llm = streamFrom([ + { type: "text-delta", index: 0, text: "ok" }, + { type: "finish", reason: { kind: "stop" } }, + ], undefined, [], undefined, () => { + resolutions++; + }); + const bridge = createDeepSeekHarnessHostLlmBridge({ llm, routes }); + const complete = () => routes.run( + { provider: "openai", model: "gpt-test" }, + () => bridge.complete({ messages: [{ role: "user", content: "hello" }] }), + ); + + await Promise.all([complete(), complete()]); + + expect(resolutions).toBe(1); + }); + + it("expires cached capabilities so silent model updates are eventually observed", async () => { + let now = 1_000; + let supportsOff = false; + let resolutions = 0; + const nowSpy = vi.spyOn(Date, "now").mockImplementation(() => now); + const routes = new DeepSeekHarnessLlmRouteContext(); + const preparations: string[] = []; + const bridge = createDeepSeekHarnessHostLlmBridge({ + llm: { + async resolveModelInfo(provider, model) { + resolutions++; + return { + provider, + id: model, + name: model, + ...(supportsOff + ? { + reasoning: { + efforts: [{ id: ReasoningEffortId("off"), name: "Off" }], + }, + } + : {}), + }; + }, + async prepareCall(config) { + preparations.push(config.reasoningEffort ?? "plain"); + return preparedCall(config, () => (async function* () { + yield { type: "text-delta", index: 0, text: "ok" } as StreamChunk; + yield { type: "finish", reason: { kind: "stop" } } as StreamChunk; + })()); + }, + }, + routes, + }); + const complete = () => routes.run( + { provider: "openai", model: "gpt-test" }, + () => bridge.complete({ messages: [{ role: "user", content: "hello" }] }), + ); + + try { + await complete(); + supportsOff = true; + now += 10 * 60 * 1_000 + 1; + await complete(); + } finally { + nowSpy.mockRestore(); + } + + expect(preparations).toEqual(["plain", "off"]); + expect(resolutions).toBe(2); + }); + + it("falls back safely when HMR changes capabilities between lookup and preparation", async () => { + const preparations: string[] = []; + let firstOff = true; + let resolutions = 0; + const routes = new DeepSeekHarnessLlmRouteContext(); + const bridge = createDeepSeekHarnessHostLlmBridge({ + llm: { + resolveModelInfo(provider, model) { + resolutions++; + return Promise.resolve({ + provider, + id: model, + name: model, + reasoning: { + efforts: [{ id: ReasoningEffortId("off"), name: "Off" }], + }, + }); + }, + async prepareCall(config) { + preparations.push(config.reasoningEffort ?? "plain"); + if (config.reasoningEffort !== undefined && firstOff) { + firstOff = false; + throw new LlmError("adapter changed", "UNSUPPORTED_REASONING_EFFORT"); + } + return preparedCall(config, () => (async function* () { + yield { type: "text-delta", index: 0, text: "ok" } as StreamChunk; + yield { type: "finish", reason: { kind: "stop" } } as StreamChunk; + })()); + }, + }, + routes, + }); + const complete = () => routes.run( + { provider: "openai", model: "gpt-test" }, + () => bridge.complete({ messages: [{ role: "user", content: "hello" }] }), + ); + + await complete(); + await complete(); + + expect(preparations).toEqual(["off", "plain", "plain"]); + expect(resolutions).toBe(1); + }); + it("does not fall back for errors other than unsupported reasoning effort", async () => { const failure = new LlmError("invalid model metadata", "INVALID_MODEL_REASONING"); const prepareCall = vi.fn(); prepareCall.mockRejectedValue(failure); const routes = new DeepSeekHarnessLlmRouteContext(); const bridge = createDeepSeekHarnessHostLlmBridge({ - llm: { prepareCall }, + llm: { + resolveModelInfo: (provider, model) => Promise.resolve({ + provider, + id: model, + name: model, + reasoning: { + efforts: [{ id: ReasoningEffortId("off"), name: "Off" }], + }, + }), + prepareCall, + }, routes, }); @@ -253,6 +450,9 @@ describe("DeepSeek Harness host LLM bridge", () => { const routes = new DeepSeekHarnessLlmRouteContext(); const bridge = createDeepSeekHarnessHostLlmBridge({ llm: { + resolveModelInfo: (provider, model, signal) => ( + ctx.llm.resolveModelInfo(provider, model, signal) + ), async prepareCall(config, signal) { const prepared = await ctx.llm.prepareCall(config, signal); // Simulate a provider plugin HMR swap in the exact TOCTOU window @@ -452,7 +652,14 @@ describe("DeepSeek Harness host LLM bridge", () => { const prepareCall = vi.fn(); const routes = new DeepSeekHarnessLlmRouteContext(); const bridge = createDeepSeekHarnessHostLlmBridge({ - llm: { prepareCall }, + llm: { + resolveModelInfo: (provider, model) => Promise.resolve({ + provider, + id: model, + name: model, + }), + prepareCall, + }, routes, }); const controller = new AbortController(); @@ -469,6 +676,9 @@ describe("DeepSeek Harness host LLM bridge", () => { it("enforces the MemOS timeout through the fused DSH request signal", async () => { let observedSignal: AbortSignal | undefined; const llm: DeepSeekHarnessLlmLike = { + resolveModelInfo(provider, model) { + return Promise.resolve({ provider, id: model, name: model }); + }, prepareCall(config, signal) { observedSignal = signal; return Promise.resolve(preparedCall(config, (options) => { diff --git a/apps/memos-local-plugin/tests/unit/adapters/deepseek-harness-runtime.test.ts b/apps/memos-local-plugin/tests/unit/adapters/deepseek-harness-runtime.test.ts index cc79c5045..7d6c515ef 100644 --- a/apps/memos-local-plugin/tests/unit/adapters/deepseek-harness-runtime.test.ts +++ b/apps/memos-local-plugin/tests/unit/adapters/deepseek-harness-runtime.test.ts @@ -1,6 +1,7 @@ import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import type { Context } from "@deepseek-ai/cordis"; +import { describe, expect, it, vi } from "vitest"; import { configureDeepSeekHarnessHostLlm, @@ -8,7 +9,11 @@ import { deepSeekHarnessMemoryGuidance, defaultDeepSeekHarnessHome, inject, + registerDeepSeekHarnessHostLlmCapabilityInvalidation, } from "../../../adapters/deepseek-harness/index.js"; +import type { + DeepSeekHarnessHostLlmBridge, +} from "../../../adapters/deepseek-harness/host-llm.js"; import { DEFAULT_CONFIG } from "../../../core/config/index.js"; describe("DeepSeek Harness adapter runtime defaults", () => { @@ -80,4 +85,29 @@ describe("DeepSeek Harness adapter runtime defaults", () => { expect(withoutTools).toContain("untrusted historical data"); expect(deepSeekHarnessMemoryGuidance(true)).toContain("memos_search"); }); + + it("invalidates host LLM capabilities when DSH adapters are updated", () => { + let listener: (() => void) | undefined; + const unregister = vi.fn(); + const ctx = { + on(event: string, callback: () => void): () => void { + expect(event).toBe("llm/adapters-updated"); + listener = callback; + return unregister; + }, + } as unknown as Context; + const invalidateModelCapabilities = vi.fn(); + const bridge = { + invalidateModelCapabilities, + } as unknown as DeepSeekHarnessHostLlmBridge; + + const registered = registerDeepSeekHarnessHostLlmCapabilityInvalidation( + ctx, + bridge, + ); + listener?.(); + + expect(registered).toBe(unregister); + expect(invalidateModelCapabilities).toHaveBeenCalledOnce(); + }); });