From 7363173679ccb0bde30c0027636a5e7c210a7dcf Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Mon, 31 Aug 2026 22:53:32 +0530 Subject: [PATCH 01/41] feat(site-memory): resolve product identities safely --- package-lock.json | 15 ++-- package.json | 1 + src/site-memory/model.ts | 72 +++++++++++++++++++ src/site-memory/product-resolver.test.ts | 88 ++++++++++++++++++++++++ src/site-memory/product-resolver.ts | 68 ++++++++++++++++++ 5 files changed, 237 insertions(+), 7 deletions(-) create mode 100644 src/site-memory/model.ts create mode 100644 src/site-memory/product-resolver.test.ts create mode 100644 src/site-memory/product-resolver.ts diff --git a/package-lock.json b/package-lock.json index 4c050b39..9bca9b32 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "jsdom": "^29.0.2", "playwright-core": "1.61.1", "quickjs-emscripten": "0.32.0", + "tldts": "^7.4.11", "turndown": "^7.2.2", "turndown-plugin-gfm": "^1.0.2", "undici": "^6.27.0", @@ -2990,21 +2991,21 @@ } }, "node_modules/tldts": { - "version": "7.0.28", - "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.28.tgz", - "integrity": "sha512-+Zg3vWhRUv8B1maGSTFdev9mjoo8Etn2Ayfs4cnjlD3CsGkxXX4QyW3j2WJ0wdjYcYmy7Lx2RDsZMhgCWafKIw==", + "version": "7.4.11", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.4.11.tgz", + "integrity": "sha512-aBiNayCfTQxuIJBm06M+xR14cYaYlDlSXZbgsnKzKNxDKUVq7KFwTjwBSsb7m9Y5xO8WfPnBc63WaYFMTGlvqw==", "license": "MIT", "dependencies": { - "tldts-core": "^7.0.28" + "tldts-core": "^7.4.11" }, "bin": { "tldts": "bin/cli.js" } }, "node_modules/tldts-core": { - "version": "7.0.28", - "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.28.tgz", - "integrity": "sha512-7W5Efjhsc3chVdFhqtaU0KtK32J37Zcr9RKtID54nG+tIpcY79CQK/veYPODxtD/LJ4Lue66jvrQzIX2Z2/pUQ==", + "version": "7.4.11", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.4.11.tgz", + "integrity": "sha512-CW3WN2rIIE/Of21mulhgnGOwoDyEFNygyIBOONSdyAuSATgMMUCpLeUlB+E8sAwA5xRV9hYPl+kyZ9citHCaKg==", "license": "MIT" }, "node_modules/tough-cookie": { diff --git a/package.json b/package.json index 9bec7da9..2bab19c2 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,7 @@ "jsdom": "^29.0.2", "playwright-core": "1.61.1", "quickjs-emscripten": "0.32.0", + "tldts": "^7.4.11", "turndown": "^7.2.2", "turndown-plugin-gfm": "^1.0.2", "undici": "^6.27.0", diff --git a/src/site-memory/model.ts b/src/site-memory/model.ts new file mode 100644 index 00000000..b5074e15 --- /dev/null +++ b/src/site-memory/model.ts @@ -0,0 +1,72 @@ +export type MemoryRevision = string; + +export interface ProductIdentity { + /** Filesystem-safe ASCII IDNA hostname used as the product directory key. */ + key: string; + hostname: string; + /** Unicode hostname retained for human-facing manifest output. */ + displayHostname: string; + /** PSL-aware registrable-domain boundary for provisional fallback lookup. */ + registrableDomain: string; +} + +export interface SeedPayload { + revision: string; + site: string; + references?: Record; +} + +export type SeedLookupResult = + | { status: 'unattempted' } + | { status: 'absent' } + | { status: 'lookup-failed' } + | ({ status: 'available' } & SeedPayload); + +export interface ProductManifest { + schemaVersion: 1; + product: ProductIdentity; + /** Confirmed alternate hostnames belonging to this product. */ + interfaces: ProductIdentity[]; + seed: SeedLookupResult; +} + +export type ProductResolutionStatus = 'exact' | 'confirmed-interface' | 'provisional-fallback' | 'new'; + +export interface ProductResolution { + status: ProductResolutionStatus; + requested: ProductIdentity; + product: ProductIdentity; + manifest?: ProductManifest; + /** Only a provisional parent fallback is barred from writes. */ + readOnly: boolean; +} + +export type CandidateStatus = 'pending' | 'ingested' | 'rejected'; + +export interface CandidateEnvironment { + machine?: string; + localIp?: string; + publicIp?: string; + os?: string; + browserVersion?: string; + webcmdVersion?: string; +} + +export interface Candidate { + schemaVersion: 1; + id: string; + domain: string; + hostname: string; + observedAt: string; + observedDateUtc: string; + kind: string; + claim: string; + evidence: string; + consequence: string; + environment: CandidateEnvironment; + status: CandidateStatus; + evidenceRole: 'supporting' | 'dissenting' | null; + memoryCommit: MemoryRevision | null; + reviewedAt: string | null; + rejectionReason: string | null; +} diff --git a/src/site-memory/product-resolver.test.ts b/src/site-memory/product-resolver.test.ts new file mode 100644 index 00000000..b430a13e --- /dev/null +++ b/src/site-memory/product-resolver.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from 'vitest'; +import type { ProductManifest } from './model.js'; +import { canonicalProductKey, resolveProduct } from './product-resolver.js'; + +const manifest = (url: string, interfaces: string[] = []): ProductManifest => ({ + schemaVersion: 1, + product: canonicalProductKey(url), + interfaces: interfaces.map(canonicalProductKey), + seed: { status: 'unattempted' }, +}); + +describe('product identity resolution', () => { + it('uses lowercase ASCII IDNA keys while retaining a Unicode display hostname', () => { + expect(canonicalProductKey('https://BÜCHER.Example/Books')).toEqual({ + key: 'xn--bcher-kva.example', + hostname: 'xn--bcher-kva.example', + displayHostname: 'bücher.example', + registrableDomain: 'xn--bcher-kva.example', + }); + }); + + it.each([ + '', + '../example.test', + 'example.test/../private', + 'https://127.0.0.1/', + 'https://[::1]/', + 'localhost', + 'https://example.test@evil.test/', + 'https://example.test\\private', + ])('rejects non-contained host input %j', (input) => { + expect(() => canonicalProductKey(input)).toThrow(/Invalid product hostname/); + }); + + it('uses an exact product manifest before a parent fallback', () => { + const resolution = resolveProduct('https://old.reddit.com/r/typescript', [ + manifest('reddit.com'), + manifest('old.reddit.com'), + ]); + + expect(resolution).toMatchObject({ + status: 'exact', + readOnly: false, + product: { key: 'old.reddit.com' }, + }); + }); + + it('returns the registrable-domain product as a read-only provisional fallback', () => { + const resolution = resolveProduct('https://old.reddit.com/r/typescript', [manifest('reddit.com')]); + + expect(resolution).toMatchObject({ + status: 'provisional-fallback', + readOnly: true, + product: { key: 'reddit.com' }, + requested: { key: 'old.reddit.com' }, + }); + }); + + it('does not look beyond the PSL-aware registrable-domain boundary', () => { + const resolution = resolveProduct('https://news.ycombinator.com/', [manifest('ycombinator.com')]); + + expect(resolution).toMatchObject({ + status: 'provisional-fallback', + product: { key: 'ycombinator.com' }, + requested: { key: 'news.ycombinator.com', registrableDomain: 'ycombinator.com' }, + }); + }); + + it('distinguishes a confirmed interface from a distinct exact product', () => { + const interfaceResolution = resolveProduct('https://old.reddit.com/', [manifest('reddit.com', ['old.reddit.com'])]); + const productResolution = resolveProduct('https://news.ycombinator.com/', [ + manifest('ycombinator.com', ['www.ycombinator.com']), + manifest('news.ycombinator.com'), + ]); + + expect(interfaceResolution).toMatchObject({ + status: 'confirmed-interface', + readOnly: false, + product: { key: 'reddit.com' }, + requested: { key: 'old.reddit.com' }, + }); + expect(productResolution).toMatchObject({ + status: 'exact', + readOnly: false, + product: { key: 'news.ycombinator.com' }, + }); + }); +}); diff --git a/src/site-memory/product-resolver.ts b/src/site-memory/product-resolver.ts new file mode 100644 index 00000000..934fbf51 --- /dev/null +++ b/src/site-memory/product-resolver.ts @@ -0,0 +1,68 @@ +import { isIP } from 'node:net'; +import { domainToUnicode } from 'node:url'; +import { getDomain } from 'tldts'; +import type { ProductIdentity, ProductManifest, ProductResolution } from './model.js'; + +export function canonicalProductKey(urlOrHost: string): ProductIdentity { + const value = urlOrHost.trim(); + if (!value || value.includes('\\')) throw invalidHost(urlOrHost); + + const url = parseUrlOrHost(value); + if (url.username || url.password || !['http:', 'https:'].includes(url.protocol)) throw invalidHost(urlOrHost); + + const hostname = url.hostname.toLowerCase(); + const labels = hostname.split('.'); + if (!hostname || isIP(hostname.replace(/^\[|\]$/g, '')) || labels.some((label) => !label || label === '.' || label === '..')) { + throw invalidHost(urlOrHost); + } + + const registrableDomain = getDomain(hostname, { allowPrivateDomains: true }); + if (!registrableDomain) throw invalidHost(urlOrHost); + + return { + key: hostname, + hostname, + displayHostname: domainToUnicode(hostname) || hostname, + registrableDomain, + }; +} + +export function resolveProduct(url: string, manifests: ProductManifest[]): ProductResolution { + const requested = canonicalProductKey(url); + const exact = manifests.find((manifest) => manifest.product.key === requested.key); + if (exact) return resolved('exact', requested, exact.product, exact, false); + + const interfaceManifest = manifests.find((manifest) => manifest.interfaces.some(({ key }) => key === requested.key)); + if (interfaceManifest) return resolved('confirmed-interface', requested, interfaceManifest.product, interfaceManifest, false); + + const parent = requested.key === requested.registrableDomain + ? undefined + : manifests.find((manifest) => manifest.product.key === requested.registrableDomain); + if (parent) return resolved('provisional-fallback', requested, parent.product, parent, true); + + return resolved('new', requested, requested, undefined, false); +} + +function parseUrlOrHost(value: string): URL { + try { + if (value.includes('://')) return new URL(value); + if (value.includes('/') || value.includes('..')) throw invalidHost(value); + return new URL(`https://${value}`); + } catch { + throw invalidHost(value); + } +} + +function invalidHost(value: string): Error { + return new Error(`Invalid product hostname: ${value}`); +} + +function resolved( + status: ProductResolution['status'], + requested: ProductIdentity, + product: ProductIdentity, + manifest: ProductManifest | undefined, + readOnly: boolean, +): ProductResolution { + return { status, requested, product, ...(manifest ? { manifest } : {}), readOnly }; +} From d95196543d77dcca190e21b4b3f9ee255f76729b Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Mon, 31 Aug 2026 22:59:12 +0530 Subject: [PATCH 02/41] fix(site-memory): fall back to nearest parent product Walk parent hostnames to the PSL registrable-domain boundary instead of jumping to the apex, so admin.eu.example.com prefers eu.example.com. --- src/site-memory/product-resolver.test.ts | 14 ++++++++++++++ src/site-memory/product-resolver.ts | 10 ++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/site-memory/product-resolver.test.ts b/src/site-memory/product-resolver.test.ts index b430a13e..d58879e4 100644 --- a/src/site-memory/product-resolver.test.ts +++ b/src/site-memory/product-resolver.test.ts @@ -56,6 +56,20 @@ describe('product identity resolution', () => { }); }); + it('chooses the nearest existing parent inside the registrable-domain boundary', () => { + const resolution = resolveProduct('https://admin.eu.example.com/', [ + manifest('example.com'), + manifest('eu.example.com'), + ]); + + expect(resolution).toMatchObject({ + status: 'provisional-fallback', + readOnly: true, + product: { key: 'eu.example.com' }, + requested: { key: 'admin.eu.example.com' }, + }); + }); + it('does not look beyond the PSL-aware registrable-domain boundary', () => { const resolution = resolveProduct('https://news.ycombinator.com/', [manifest('ycombinator.com')]); diff --git a/src/site-memory/product-resolver.ts b/src/site-memory/product-resolver.ts index 934fbf51..0fa0d29d 100644 --- a/src/site-memory/product-resolver.ts +++ b/src/site-memory/product-resolver.ts @@ -35,10 +35,12 @@ export function resolveProduct(url: string, manifests: ProductManifest[]): Produ const interfaceManifest = manifests.find((manifest) => manifest.interfaces.some(({ key }) => key === requested.key)); if (interfaceManifest) return resolved('confirmed-interface', requested, interfaceManifest.product, interfaceManifest, false); - const parent = requested.key === requested.registrableDomain - ? undefined - : manifests.find((manifest) => manifest.product.key === requested.registrableDomain); - if (parent) return resolved('provisional-fallback', requested, parent.product, parent, true); + const labels = requested.key.split('.'); + const boundary = requested.registrableDomain.split('.').length; + for (let i = 1; i <= labels.length - boundary; i++) { + const parent = manifests.find((manifest) => manifest.product.key === labels.slice(i).join('.')); + if (parent) return resolved('provisional-fallback', requested, parent.product, parent, true); + } return resolved('new', requested, requested, undefined, false); } From dc84ad15524d221f3aca610cc1858bcebaacddd1 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Mon, 31 Aug 2026 23:11:32 +0530 Subject: [PATCH 03/41] feat(site-memory): add safe local git transactions --- src/site-memory/file-lock.ts | 4 + src/site-memory/git-store.test.ts | 185 ++++++++++++++++++++++++++++ src/site-memory/git-store.ts | 135 ++++++++++++++++++++ src/site-memory/local-store.test.ts | 43 +++++++ src/site-memory/local-store.ts | 61 ++++++++- 5 files changed, 425 insertions(+), 3 deletions(-) create mode 100644 src/site-memory/git-store.test.ts create mode 100644 src/site-memory/git-store.ts diff --git a/src/site-memory/file-lock.ts b/src/site-memory/file-lock.ts index eae2265f..fed16caa 100644 --- a/src/site-memory/file-lock.ts +++ b/src/site-memory/file-lock.ts @@ -26,6 +26,10 @@ import { isActionablePid, isPidAlive } from '../session-lease.js'; export const LOCK_STALE_MS = 10_000; /** Total acquire budget. Longer than LOCK_STALE_MS so stale locks are broken, not reported. */ export const LOCK_TIMEOUT_MS = 15_000; +/** Repository lock stale bound: copy, explicit staging, and two local commits, including slow Git. */ +export const REPOSITORY_LOCK_STALE_MS = 60_000; +/** Total repository-lock acquire budget. Longer than REPOSITORY_LOCK_STALE_MS so stale owners recover. */ +export const REPOSITORY_LOCK_TIMEOUT_MS = 90_000; const RETRY_MIN_MS = 5; const RETRY_MAX_MS = 50; diff --git a/src/site-memory/git-store.test.ts b/src/site-memory/git-store.test.ts new file mode 100644 index 00000000..5af2d009 --- /dev/null +++ b/src/site-memory/git-store.test.ts @@ -0,0 +1,185 @@ +import { execFile, spawnSync } from 'node:child_process'; +import { chmod, mkdir, mkdtemp, realpath, rm, utimes, writeFile } from 'node:fs/promises'; +import { hostname, tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + LOCK_STALE_MS, + LOCK_TIMEOUT_MS, + lockPathFor, + REPOSITORY_LOCK_STALE_MS, + REPOSITORY_LOCK_TIMEOUT_MS, +} from './file-lock.js'; +import { openSitesRepository } from './git-store.js'; +import { listProductKeys, writeProductFile } from './local-store.js'; + +const run = promisify(execFile); +const tempHomes: string[] = []; +const originalPath = process.env.PATH; + +afterEach(async () => { + process.env.PATH = originalPath; + await Promise.all(tempHomes.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +describe('sites git repository', () => { + it('initializes on first commit with a dedicated local author and no remote', async () => { + const { homeDir, sites } = await tempSites(); + await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); + + const repo = await openSitesRepository({ homeDir }); + const revision = await repo.commit(['example.test/manifest.json'], 'init example.test'); + + expect(revision).toMatch(/^[0-9a-f]{40}$/); + expect(await repo.revision()).toBe(revision); + expect((await git(sites, ['config', '--local', 'user.name'])).trim()).toBe('webcmd'); + expect((await git(sites, ['config', '--local', 'user.email'])).trim()).toBe('webcmd@local'); + expect((await git(sites, ['log', '-1', '--format=%an <%ae>'])).trim()).toBe('webcmd '); + expect((await git(sites, ['remote'])).trim()).toBe(''); + }); + + it('creates a gitignore for drafts, locks, temps, fixtures, and verify artifacts', async () => { + const { homeDir, sites } = await tempSites(); + await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); + + await (await openSitesRepository({ homeDir })).commit(['example.test/manifest.json'], 'init'); + + const ignore = await git(sites, ['show', 'HEAD:.gitignore']); + expect(ignore).toMatch(/^\.drafts\/$/m); + expect(ignore).toMatch(/\*\.lock/); + expect(ignore).toMatch(/\*\.tmp/); + expect(ignore).toMatch(/fixtures/); + expect(ignore).toMatch(/verify/); + }); + + it('accepts a repository whose real toplevel is exactly the sites root', async () => { + const { homeDir, sites } = await tempSites(); + await mkdir(sites, { recursive: true }); + await git(sites, ['init']); + await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); + + const revision = await (await openSitesRepository({ homeDir })).commit(['example.test/manifest.json'], 'init'); + + expect(await realpath((await git(sites, ['rev-parse', '--show-toplevel'])).trim())).toBe(await realpath(sites)); + expect(revision).toMatch(/^[0-9a-f]{40}$/); + }); + + it('refuses an ancestor repository that owns the sites path', async () => { + const { homeDir, sites } = await tempSites(); + await mkdir(sites, { recursive: true }); + await git(homeDir, ['init']); + + await expect(openSitesRepository({ homeDir })).rejects.toThrow(/ancestor/i); + }); + + it('refuses to commit when an unrelated dirty file exists', async () => { + const { homeDir } = await tempSites(); + await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); + const repo = await openSitesRepository({ homeDir }); + await repo.commit(['example.test/manifest.json'], 'init'); + await writeProductFile('example.test', 'manifest.json', '{"dirty":true}\n', { homeDir }); + await writeProductFile('other.test', 'manifest.json', '{}\n', { homeDir }); + + await expect(repo.commit(['other.test/manifest.json'], 'other')).rejects.toThrow(/unrelated/i); + }); + + it('stages only explicit paths and never git add .', async () => { + const { homeDir, sites } = await tempSites(); + await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); + await writeProductFile('example.test', 'fixtures/sample.json', '{"ok":true}\n', { homeDir }); + await mkdir(join(sites, '.drafts', 'task'), { recursive: true }); + await writeFile(join(sites, '.drafts', 'task', 'scratch.md'), 'draft'); + + await (await openSitesRepository({ homeDir })).commit(['example.test/manifest.json'], 'init'); + + const files = (await git(sites, ['ls-files'])).trim().split('\n').sort(); + expect(files).toEqual(['.gitignore', 'example.test/manifest.json']); + }); + + it('serializes concurrent commits so both explicit writes survive', async () => { + const { homeDir, sites } = await tempSites(); + const repo = await openSitesRepository({ homeDir }); + await writeProductFile('a.test', 'manifest.json', 'a\n', { homeDir }); + await writeProductFile('b.test', 'manifest.json', 'b\n', { homeDir }); + + await Promise.all([ + repo.commit(['a.test/manifest.json'], 'a'), + repo.commit(['b.test/manifest.json'], 'b'), + ]); + + const files = (await git(sites, ['ls-files'])).trim().split('\n').sort(); + expect(files).toEqual(['.gitignore', 'a.test/manifest.json', 'b.test/manifest.json']); + }); + + it('keeps a slow git commit inside the repository-specific stale bound', async () => { + expect(REPOSITORY_LOCK_STALE_MS).toBeGreaterThan(LOCK_STALE_MS); + expect(REPOSITORY_LOCK_TIMEOUT_MS).toBeGreaterThan(LOCK_TIMEOUT_MS); + + const { homeDir } = await tempSites(); + await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); + await installSlowGit(250); + const started = Date.now(); + + const revision = await (await openSitesRepository({ homeDir })).commit(['example.test/manifest.json'], 'slow'); + + expect(Date.now() - started).toBeGreaterThanOrEqual(250); + expect(revision).toMatch(/^[0-9a-f]{40}$/); + }); + + it('recovers a repository lock left by a stale owner', async () => { + const { homeDir, sites } = await tempSites(); + await mkdir(sites, { recursive: true }); + const lockPath = lockPathFor(join(sites, '.repository')); + await writeFile(lockPath, `${JSON.stringify({ pid: deadPid(), host: hostname(), token: 'stale' })}\n`); + const past = new Date(Date.now() - REPOSITORY_LOCK_STALE_MS - 1_000); + await utimes(lockPath, past, past); + + const repo = await openSitesRepository({ homeDir }); + await expect(repo.withRepositoryLock(async () => 'ok')).resolves.toBe('ok'); + }); + + it('excludes .git, .drafts, and other dot entries from product enumeration', async () => { + const { homeDir, sites } = await tempSites(); + await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); + await mkdir(join(sites, '.drafts', 'task'), { recursive: true }); + await mkdir(join(sites, '.cache'), { recursive: true }); + await writeFile(join(sites, '.hidden'), 'nope'); + await (await openSitesRepository({ homeDir })).commit(['example.test/manifest.json'], 'init'); + + await expect(listProductKeys({ homeDir })).resolves.toEqual(['example.test']); + }); +}); + +async function tempSites() { + const homeDir = await mkdtemp(join(tmpdir(), 'webcmd-git-store-')); + tempHomes.push(homeDir); + return { homeDir, sites: join(homeDir, '.webcmd', 'sites') }; +} + +async function git(cwd: string, args: string[]) { + const { stdout } = await run('git', args, { cwd, encoding: 'utf8' }); + return stdout; +} + +async function installSlowGit(delayMs: number) { + const dir = await mkdtemp(join(tmpdir(), 'webcmd-slow-git-')); + tempHomes.push(dir); + const { stdout } = await run('/usr/bin/which', ['git'], { encoding: 'utf8' }); + const wrapper = join(dir, 'git'); + await writeFile(wrapper, `#!/usr/bin/env node +const { spawnSync } = require('node:child_process'); +const args = process.argv.slice(2); +if (args.includes('commit')) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ${delayMs}); +const result = spawnSync(${JSON.stringify(stdout.trim())}, args, { stdio: 'inherit' }); +process.exit(result.status ?? 1); +`); + await chmod(wrapper, 0o755); + process.env.PATH = `${dir}:${process.env.PATH}`; +} + +function deadPid(): number { + const child = spawnSync(process.execPath, ['-e', '']); + if (typeof child.pid !== 'number') throw new Error('could not spawn a child to retire'); + return child.pid; +} diff --git a/src/site-memory/git-store.ts b/src/site-memory/git-store.ts new file mode 100644 index 00000000..5cb4930f --- /dev/null +++ b/src/site-memory/git-store.ts @@ -0,0 +1,135 @@ +import { execFile as execFileCb } from 'node:child_process'; +import { mkdir, realpath } from 'node:fs/promises'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import { REPOSITORY_LOCK_STALE_MS, REPOSITORY_LOCK_TIMEOUT_MS, withFileLock } from './file-lock.js'; +import { atomicWrite, containedRelativePath, sitesRoot, type LocalStoreOptions } from './local-store.js'; +import type { MemoryRevision } from './model.js'; + +const execFile = promisify(execFileCb); +const AUTHOR_NAME = 'webcmd'; +const AUTHOR_EMAIL = 'webcmd@local'; +const GITIGNORE = ['.drafts/', '*.lock', '*.tmp', '**/fixtures/', '**/verify/', ''].join('\n'); +const GIT_FLAGS = [ + '-c', `user.name=${AUTHOR_NAME}`, + '-c', `user.email=${AUTHOR_EMAIL}`, + '-c', 'commit.gpgsign=false', + '-c', 'core.hooksPath=/dev/null', +]; + +export interface SitesRepository { + revision(): Promise; + commit(paths: string[], message: string): Promise; + withRepositoryLock(fn: () => Promise): Promise; +} + +export async function openSitesRepository(options: LocalStoreOptions = {}): Promise { + const root = await ensureSitesRoot(options); + await assertExactRootOrAbsent(root); + return { + revision: () => revisionOf(root), + commit: (paths, message) => withRepositoryLock(root, () => commitPaths(root, paths, message)), + withRepositoryLock: (fn) => withRepositoryLock(root, fn), + }; +} + +function withRepositoryLock(root: string, fn: () => Promise): Promise { + return withFileLock(join(root, '.repository'), fn, { + staleMs: REPOSITORY_LOCK_STALE_MS, + timeoutMs: REPOSITORY_LOCK_TIMEOUT_MS, + }); +} + +async function ensureSitesRoot(options: LocalStoreOptions): Promise { + const root = sitesRoot(options); + await mkdir(root, { recursive: true }); + return realpath(root); +} + +async function commitPaths(root: string, paths: string[], message: string): Promise { + if (paths.length === 0) throw new Error('Refusing to commit without explicit paths.'); + await ensureRepository(root); + const relativePaths = paths.map((path) => containedRelativePath(root, path)); + await atomicWrite(join(root, '.gitignore'), GITIGNORE); + await assertNoUnrelatedDirty(root, relativePaths); + await git(root, ['add', '--', ...relativePaths, '.gitignore']); + await git(root, ['commit', '--no-gpg-sign', '-m', message]); + return (await git(root, ['rev-parse', 'HEAD'])).trim(); +} + +async function ensureRepository(root: string): Promise { + if (!await hasExactRepository(root)) { + await git(root, ['init']); + await assertExactRoot(root); + } + await git(root, ['config', 'user.name', AUTHOR_NAME]); + await git(root, ['config', 'user.email', AUTHOR_EMAIL]); +} + +async function revisionOf(root: string): Promise { + try { + return (await git(root, ['rev-parse', 'HEAD'])).trim(); + } catch { + return null; + } +} + +async function assertExactRootOrAbsent(root: string): Promise { + try { + await assertExactRoot(root); + } catch (err) { + if (isNotRepo(err)) return; + throw err; + } +} + +async function hasExactRepository(root: string): Promise { + try { + await assertExactRoot(root); + return true; + } catch (err) { + if (isNotRepo(err)) return false; + throw err; + } +} + +async function assertExactRoot(root: string): Promise { + const top = (await git(root, ['rev-parse', '--show-toplevel'])).trim(); + if (await realpath(top) !== await realpath(root)) { + throw new Error('An ancestor Git repository owns the sites path; it is not the sites repository.'); + } +} + +async function assertNoUnrelatedDirty(root: string, allowed: string[]): Promise { + const allow = new Set([...allowed, '.gitignore']); + const status = await git(root, ['status', '--porcelain', '-uall']); + for (const line of status.split('\n').filter(Boolean)) { + const code = line.slice(0, 2); + if (code === '??' || code === '!!') continue; + const path = porcelainPath(line); + if (!allow.has(path)) throw new Error(`Refusing to commit unrelated dirty path: ${path}`); + } +} + +function porcelainPath(line: string): string { + const renamed = line.indexOf(' -> '); + return renamed === -1 ? line.slice(3) : line.slice(renamed + 4); +} + +async function git(root: string, args: string[]): Promise { + const env = { ...process.env }; + delete env.GIT_DIR; + delete env.GIT_WORK_TREE; + env.GIT_AUTHOR_NAME = AUTHOR_NAME; + env.GIT_AUTHOR_EMAIL = AUTHOR_EMAIL; + env.GIT_COMMITTER_NAME = AUTHOR_NAME; + env.GIT_COMMITTER_EMAIL = AUTHOR_EMAIL; + const { stdout } = await execFile('git', [...GIT_FLAGS, ...args], { cwd: root, encoding: 'utf8', env }); + return stdout; +} + +function isNotRepo(err: unknown): boolean { + if (!(err instanceof Error)) return false; + const stderr = 'stderr' in err && typeof err.stderr === 'string' ? err.stderr : ''; + return 'code' in err && err.code === 128 && /not a git repository/i.test(stderr); +} diff --git a/src/site-memory/local-store.test.ts b/src/site-memory/local-store.test.ts index 7f514d55..633c888b 100644 --- a/src/site-memory/local-store.test.ts +++ b/src/site-memory/local-store.test.ts @@ -11,12 +11,16 @@ import { addFieldMapping, addResponseSample, appendNote, + copyDraftFiles, getVerifyFixture, + listProductKeys, listSiteMemory, markEndpointStale, putVerifyFixture, + readProductFile, setEndpoint, showSiteMemory, + writeProductFile, } from './local-store.js'; const tempHomes: string[] = []; @@ -262,6 +266,45 @@ describe('local site memory store', () => { await expect(listSiteMemory(base.site, { homeDir, paths: ['linkdir/secret.txt'] })).rejects.toThrow(/Invalid site memory path/); }); + it('writes and reads a contained product file', async () => { + const homeDir = await tempHome(); + await writeProductFile('example.test', 'sitemap/SITE.md', '# Example\n', { homeDir }); + + await expect(readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).resolves.toBe('# Example\n'); + }); + + it('rejects a product path that escapes the sites root', async () => { + const homeDir = await tempHome(); + + await expect(writeProductFile('example.test', '../outside.md', 'nope\n', { homeDir })) + .rejects.toThrow(/Invalid site memory path/); + await expect(readProductFile('example.test', '../outside.md', { homeDir })) + .rejects.toThrow(/Invalid site memory path/); + }); + + it('copies contained draft files into product memory', async () => { + const homeDir = await tempHome(); + const draft = join(homeDir, '.webcmd/sites/.drafts/task-1/example.test/sitemap/SITE.md'); + await mkdir(join(draft, '..'), { recursive: true }); + await writeFile(draft, '# Draft\n'); + + await copyDraftFiles('example.test', 'task-1', ['sitemap/SITE.md'], { homeDir }); + + await expect(readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).resolves.toBe('# Draft\n'); + }); + + it('lists product keys while skipping .git, .drafts, and other dot entries', async () => { + const homeDir = await tempHome(); + await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); + const sites = join(homeDir, '.webcmd/sites'); + await mkdir(join(sites, '.git'), { recursive: true }); + await mkdir(join(sites, '.drafts', 'task'), { recursive: true }); + await mkdir(join(sites, '.cache'), { recursive: true }); + await writeFile(join(sites, '.hidden'), 'nope'); + + await expect(listProductKeys({ homeDir })).resolves.toEqual(['example.test']); + }); + it('uses USERPROFILE instead of writing under cwd when HOME is unset', async () => { const homeDir = await tempHome(); delete process.env.HOME; diff --git a/src/site-memory/local-store.ts b/src/site-memory/local-store.ts index 012da86c..134b2fe8 100644 --- a/src/site-memory/local-store.ts +++ b/src/site-memory/local-store.ts @@ -174,6 +174,54 @@ async function updateJson( }); } +export async function readProductFile(productKey: string, path: string, opts: LocalStoreOptions = {}): Promise { + const productRoot = join(sitesRoot(opts), productSegment(productKey)); + const relative = containedRelativePath(productRoot, path); + if (!await exists(productRoot)) return null; + try { + const safe = await readableRelativePath(productRoot, relative); + return await readFile(join(productRoot, safe), 'utf8'); + } catch (err) { + if (isNodeError(err) && err.code === 'ENOENT') return null; + throw err; + } +} + +export async function writeProductFile(productKey: string, path: string, body: string, opts: LocalStoreOptions = {}): Promise { + const productRoot = join(sitesRoot(opts), productSegment(productKey)); + await mkdir(productRoot, { recursive: true }); + const relative = containedRelativePath(productRoot, path); + const target = join(productRoot, ...relative.split('/')); + await mkdir(dirname(target), { recursive: true }); + await assertInsideSiteRoot(productRoot, dirname(target), path); + await withWriteLock(target, () => atomicWrite(target, body)); +} + +export async function copyDraftFiles(productKey: string, taskId: string, paths: string[], opts: LocalStoreOptions = {}): Promise { + const draftRoot = join(sitesRoot(opts), '.drafts', productSegment(taskId), productSegment(productKey)); + for (const path of paths) { + const relative = containedRelativePath(draftRoot, path); + const safe = await readableRelativePath(draftRoot, relative); + const body = await readFile(join(draftRoot, safe), 'utf8'); + await writeProductFile(productKey, relative, body, opts); + } +} + +export async function listProductKeys(opts: LocalStoreOptions = {}): Promise { + const root = sitesRoot(opts); + if (!await exists(root)) return []; + const entries = await readdir(root, { withFileTypes: true }); + return entries.filter((entry) => entry.isDirectory() && !entry.name.startsWith('.')).map((entry) => entry.name).sort(); +} + +export function sitesRoot(opts: LocalStoreOptions = {}): string { + return join(requiredHomeDir(opts), '.webcmd', 'sites'); +} + +export function containedRelativePath(root: string, path: string): string { + return safeRelativePath(root, path).split(sep).join('/'); +} + async function writeSiteFile(site: string, path: string, body: string, opts: LocalStoreOptions): Promise { const root = await ensureSiteRoot(site, opts); const target = join(root, path); @@ -191,7 +239,7 @@ function withWriteLock(target: string, fn: () => Promise): Promise { return withPathLock(target, () => withFileLock(target, fn)); } -async function withPathLock(target: string, fn: () => Promise): Promise { +export async function withPathLock(target: string, fn: () => Promise): Promise { const previous = writeChains.get(target) ?? Promise.resolve(); const next = previous.catch(() => undefined).then(fn); const settled = next.then(() => undefined, () => undefined); @@ -202,7 +250,7 @@ async function withPathLock(target: string, fn: () => Promise): Promise return next; } -async function atomicWrite(target: string, body: string): Promise { +export async function atomicWrite(target: string, body: string): Promise { const temp = join(dirname(target), `.${basename(target)}.${process.pid}.${randomUUID()}.tmp`); try { await writeFile(temp, body, 'utf8'); @@ -236,7 +284,14 @@ function siteRoot(site: string, opts: LocalStoreOptions): string { if (!site || site.includes('/') || site.includes('\\') || site === '.' || site === '..') { throw new Error(`Invalid site memory site: ${site}`); } - return join(requiredHomeDir(opts), '.webcmd', 'sites', site); + return join(sitesRoot(opts), site); +} + +function productSegment(value: string): string { + if (!value || value.includes('/') || value.includes('\\') || value === '.' || value === '..' || value.startsWith('.')) { + throw new Error(`Invalid site memory path: ${value}`); + } + return value; } async function memoryPaths(root: string, requested?: string[]): Promise { From ff176fa22827063acf7e6d225ef9efadde6798a9 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Mon, 31 Aug 2026 23:16:42 +0530 Subject: [PATCH 04/41] fix(site-memory): reenter nested file lock in the same async chain --- src/site-memory/file-lock.ts | 9 ++++++++- src/site-memory/git-store.test.ts | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/site-memory/file-lock.ts b/src/site-memory/file-lock.ts index fed16caa..2d1df6a5 100644 --- a/src/site-memory/file-lock.ts +++ b/src/site-memory/file-lock.ts @@ -15,6 +15,7 @@ * takes milliseconds. `timeoutMs` is deliberately longer than `staleMs` so an * abandoned lock is always broken rather than surfaced to the user as an error. */ +import { AsyncLocalStorage } from 'node:async_hooks'; import { randomUUID } from 'node:crypto'; import { open, readFile, stat, unlink } from 'node:fs/promises'; import { hostname } from 'node:os'; @@ -33,6 +34,7 @@ export const REPOSITORY_LOCK_TIMEOUT_MS = 90_000; const RETRY_MIN_MS = 5; const RETRY_MAX_MS = 50; +const heldLocks = new AsyncLocalStorage>(); export interface FileLockOptions { staleMs?: number; @@ -54,9 +56,14 @@ export function lockPathFor(target: string): string { /** Run `fn` while holding the cross-process lock for `target`. */ export async function withFileLock(target: string, fn: () => Promise, options: FileLockOptions = {}): Promise { const lockPath = lockPathFor(target); + const owned = heldLocks.getStore(); + if (owned?.has(lockPath)) return fn(); + const token = await acquire(lockPath, options); + const next = new Set(owned); + next.add(lockPath); try { - return await fn(); + return await heldLocks.run(next, fn); } finally { await release(lockPath, token); } diff --git a/src/site-memory/git-store.test.ts b/src/site-memory/git-store.test.ts index 5af2d009..e36e069e 100644 --- a/src/site-memory/git-store.test.ts +++ b/src/site-memory/git-store.test.ts @@ -112,6 +112,19 @@ describe('sites git repository', () => { expect(files).toEqual(['.gitignore', 'a.test/manifest.json', 'b.test/manifest.json']); }); + it('commits from inside withRepositoryLock without deadlocking', async () => { + const { homeDir } = await tempSites(); + await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); + const repo = await openSitesRepository({ homeDir }); + + const revision = await repo.withRepositoryLock(async () => + repo.commit(['example.test/manifest.json'], 'nested'), + ); + + expect(revision).toMatch(/^[0-9a-f]{40}$/); + expect(await repo.revision()).toBe(revision); + }, 5_000); + it('keeps a slow git commit inside the repository-specific stale bound', async () => { expect(REPOSITORY_LOCK_STALE_MS).toBeGreaterThan(LOCK_STALE_MS); expect(REPOSITORY_LOCK_TIMEOUT_MS).toBeGreaterThan(LOCK_TIMEOUT_MS); From aa0a932c2a2ff400fcefeaa59c429211597f3b69 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Mon, 31 Aug 2026 23:25:10 +0530 Subject: [PATCH 05/41] feat(site-memory): initialize product context once --- src/site-memory/context.test.ts | 174 ++++++++++++++++++++++++++++ src/site-memory/context.ts | 157 +++++++++++++++++++++++++ src/site-memory/model.ts | 19 ++- src/site-memory/seed-client.test.ts | 116 +++++++++++++++++++ src/site-memory/seed-client.ts | 53 +++++++++ 5 files changed, 517 insertions(+), 2 deletions(-) create mode 100644 src/site-memory/context.test.ts create mode 100644 src/site-memory/context.ts create mode 100644 src/site-memory/seed-client.test.ts create mode 100644 src/site-memory/seed-client.ts diff --git a/src/site-memory/context.test.ts b/src/site-memory/context.test.ts new file mode 100644 index 00000000..7b96219b --- /dev/null +++ b/src/site-memory/context.test.ts @@ -0,0 +1,174 @@ +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { getMemoryContext } from './context.js'; +import { readProductFile } from './local-store.js'; +import type { GlobalSeedProvider } from './seed-client.js'; +import type { SeedLookupResult } from './model.js'; + +const run = promisify(execFile); +const tempHomes: string[] = []; + +afterEach(async () => { + await Promise.all(tempHomes.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +describe('memory context initialization', () => { + it('persists a terminal absent result without creating SITE.md', async () => { + const { homeDir, sites } = await tempSites(); + const lookup = vi.fn(async () => ({ status: 'absent' as const })); + + const context = await getMemoryContext({ + url: 'https://example.test/home', + taskId: 'task-1', + homeDir, + seedProvider: provider(lookup), + }); + + expect(context.manifest?.seed).toEqual({ status: 'absent' }); + expect(context.siteMarkdown).toBeNull(); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBeNull(); + expect(await git(sites, ['ls-files'])).toContain('example.test/manifest.json'); + expect(await git(sites, ['ls-files'])).not.toContain('example.test/sitemap/SITE.md'); + expect(lookup).toHaveBeenCalledTimes(1); + }); + + it('persists lookup-failed without creating SITE.md', async () => { + const { homeDir } = await tempSites(); + + const context = await getMemoryContext({ + url: 'https://example.test/', + taskId: 'task-1', + homeDir, + seedProvider: provider(async () => ({ status: 'lookup-failed' })), + }); + + expect(context.manifest?.seed).toEqual({ status: 'lookup-failed' }); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBeNull(); + }); + + it('does not look up a seed once a product already exists', async () => { + const { homeDir } = await tempSites(); + const lookup = vi.fn(async () => ({ status: 'absent' as const })); + const seedProvider = provider(lookup); + + await getMemoryContext({ url: 'https://example.test/', taskId: 'task-1', homeDir, seedProvider }); + lookup.mockClear(); + + const context = await getMemoryContext({ url: 'https://example.test/', taskId: 'task-2', homeDir, seedProvider }); + + expect(lookup).not.toHaveBeenCalled(); + expect(context.manifest?.seed).toEqual({ status: 'absent' }); + }); + + it('accepts an oversized seed unchanged and does not retain the body in the manifest', async () => { + const { homeDir, sites } = await tempSites(); + const site = Array.from({ length: 501 }, (_, i) => `line ${i}`).join('\n'); + const lookup = vi.fn(async (): Promise => ({ + status: 'available', + revision: 'seed-big', + site, + references: { 'alt.md': '# Alt\n' }, + })); + + const context = await getMemoryContext({ + url: 'https://example.test/', + taskId: 'task-1', + homeDir, + seedProvider: provider(lookup), + }); + + const stored = await readProductFile('example.test', 'sitemap/SITE.md', { homeDir }); + expect(stored?.split('\n').filter(Boolean)).toHaveLength(501); + expect(await readProductFile('example.test', 'sitemap/references/alt.md', { homeDir })).toBe('# Alt\n'); + expect(context.siteMarkdown).toBe(site); + expect(context.manifest?.seed).toEqual({ status: 'available', revision: 'seed-big' }); + expect(JSON.stringify(context.manifest)).not.toContain('line 0'); + expect(await git(sites, ['ls-files'])).toContain('example.test/sitemap/SITE.md'); + expect(lookup).toHaveBeenCalledTimes(1); + }); + + it('diagnoses legacy SITE.md as read-only and skips seed lookup', async () => { + const { homeDir } = await tempSites(); + await mkdir(join(homeDir, '.webcmd/sites/example.test/sitemap'), { recursive: true }); + await writeFile( + join(homeDir, '.webcmd/sites/example.test/sitemap/SITE.md'), + '---\nsite: example\nkind: site\nid: example\nstatus: verified\nverified_at: 2026-01-01\nsource: beta\n---\n# Beta\n', + ); + const lookup = vi.fn(async () => ({ status: 'available' as const, revision: 'x', site: '# no\n' })); + + const context = await getMemoryContext({ + url: 'https://example.test/', + taskId: 'task-1', + homeDir, + seedProvider: provider(lookup), + }); + + expect(context.readOnly).toBe(true); + expect(context.diagnostics.join('\n')).toMatch(/incompatible beta schema/i); + expect(lookup).not.toHaveBeenCalled(); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toMatch(/^---\n/); + }); + + it('uses a seed transiently when Git is unsafe and does not persist it', async () => { + const { homeDir } = await tempSites(); + await git(homeDir, ['init']); + const lookup = vi.fn(async (): Promise => ({ + status: 'available', + revision: 'seed-1', + site: '# Transient\n', + })); + + const context = await getMemoryContext({ + url: 'https://example.test/', + taskId: 'task-1', + homeDir, + seedProvider: provider(lookup), + }); + + expect(context.siteMarkdown).toBe('# Transient\n'); + expect(context.readOnly).toBe(true); + expect(context.manifest).toBeUndefined(); + expect(await readProductFile('example.test', 'manifest.json', { homeDir })).toBeNull(); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBeNull(); + expect(lookup).toHaveBeenCalledTimes(1); + }); + + it('creates a task-id-contained draft and rejects escaping task ids', async () => { + const { homeDir, sites } = await tempSites(); + + const context = await getMemoryContext({ + url: 'https://example.test/', + taskId: 'task-1', + homeDir, + seedProvider: provider(async () => ({ status: 'available', revision: 'r1', site: '# Draft me\n' })), + }); + + expect(context.draftPath).toBe(join(sites, '.drafts/task-1/example.test/sitemap')); + expect(await readFile(join(context.draftPath, 'SITE.md'), 'utf8')).toBe('# Draft me\n'); + await expect(getMemoryContext({ + url: 'https://example.test/', + taskId: '../escape', + homeDir, + seedProvider: provider(async () => ({ status: 'absent' })), + })).rejects.toThrow(/Invalid site memory path/); + }); +}); + +function provider(lookup: GlobalSeedProvider['lookup']): GlobalSeedProvider { + return { lookup }; +} + +async function tempSites() { + const homeDir = await mkdtemp(join(tmpdir(), 'webcmd-memory-context-')); + tempHomes.push(homeDir); + return { homeDir, sites: join(homeDir, '.webcmd', 'sites') }; +} + +async function git(cwd: string, args: string[]) { + const { stdout } = await run('git', args, { cwd, encoding: 'utf8' }); + return stdout; +} diff --git a/src/site-memory/context.ts b/src/site-memory/context.ts new file mode 100644 index 00000000..e938fba2 --- /dev/null +++ b/src/site-memory/context.ts @@ -0,0 +1,157 @@ +import { mkdir, readdir } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { openSitesRepository, type SitesRepository } from './git-store.js'; +import { atomicWrite, containedRelativePath, listProductKeys, readProductFile, sitesRoot, writeProductFile } from './local-store.js'; +import type { LocalStoreOptions } from './local-store.js'; +import type { + MemoryContext, + PersistedSeedResult, + ProductIdentity, + ProductManifest, + SeedLookupResult, +} from './model.js'; +import { canonicalProductKey, resolveProduct } from './product-resolver.js'; +import { createHttpSeedProvider, type GlobalSeedProvider } from './seed-client.js'; + +export interface MemoryContextInput extends LocalStoreOptions { + url: string; + taskId: string; + seedProvider?: GlobalSeedProvider; +} + +export async function getMemoryContext(input: MemoryContextInput): Promise { + const opts: LocalStoreOptions = { homeDir: input.homeDir }; + const taskId = memorySegment(input.taskId); + const diagnostics: string[] = []; + const git = await openGit(opts, diagnostics); + let resolution = resolveProduct(input.url, await loadManifests(opts)); + const legacy = await isLegacySite(resolution.requested.key, opts); + if (legacy) diagnostics.push('Incompatible beta schema; learning is read-only until this SITE.md is cleared.'); + + let transient: Extract | undefined; + if (resolution.status === 'new' && !legacy) { + const seed = await (input.seedProvider ?? createHttpSeedProvider()).lookup(resolution.requested.key); + if (git && seed.status !== 'unattempted') { + await persistSeed(resolution.requested, seed, git, opts); + resolution = resolveProduct(input.url, await loadManifests(opts)); + } else if (!git && seed.status === 'available') { + transient = seed; + } + } + + const productKey = resolution.product.key; + const siteMarkdown = transient?.site ?? await readProductFile(productKey, 'sitemap/SITE.md', opts); + const references = transient + ? Object.keys(transient.references ?? {}).sort().map((name) => ({ path: `sitemap/references/${name}` })) + : await listReferences(productKey, opts); + const draftPath = join(sitesRoot(opts), '.drafts', taskId, memorySegment(productKey), 'sitemap'); + if (git) await writeDraft(draftPath, productKey, siteMarkdown, references, transient?.references, opts); + + return { + resolution, + ...(resolution.manifest ? { manifest: resolution.manifest } : {}), + revision: git ? await git.revision() : null, + siteMarkdown, + references, + draftPath, + readOnly: resolution.readOnly || !git || legacy, + diagnostics, + }; +} + +async function persistSeed( + product: ProductIdentity, + seed: SeedLookupResult, + repo: SitesRepository, + opts: LocalStoreOptions, +): Promise { + const persisted: PersistedSeedResult = seed.status === 'available' ? { status: 'available', revision: seed.revision } : seed; + const manifest: ProductManifest = { schemaVersion: 1, product, interfaces: [], seed: persisted }; + const paths = [`${product.key}/manifest.json`]; + await writeProductFile(product.key, 'manifest.json', `${JSON.stringify(manifest, null, 2)}\n`, opts); + if (seed.status === 'available') { + await writeProductFile(product.key, 'sitemap/SITE.md', seed.site, opts); + paths.push(`${product.key}/sitemap/SITE.md`); + for (const [name, body] of Object.entries(seed.references ?? {})) { + const path = `sitemap/references/${name}`; + await writeProductFile(product.key, path, body, opts); + paths.push(`${product.key}/${path}`); + } + } + await repo.commit(paths, `initialize ${product.key}`); +} + +async function writeDraft( + draftPath: string, + productKey: string, + siteMarkdown: string | null, + references: { path: string }[], + transientRefs: Record | undefined, + opts: LocalStoreOptions, +): Promise { + const draftRoot = join(draftPath, '..'); + if (siteMarkdown != null) await writeContained(draftRoot, 'sitemap/SITE.md', siteMarkdown); + for (const { path } of references) { + const body = transientRefs?.[path.slice('sitemap/references/'.length)] ?? await readProductFile(productKey, path, opts); + if (body != null) await writeContained(draftRoot, path, body); + } +} + +async function writeContained(root: string, path: string, body: string): Promise { + const relative = containedRelativePath(root, path); + const target = join(root, ...relative.split('/')); + await mkdir(dirname(target), { recursive: true }); + await atomicWrite(target, body); +} + +async function loadManifests(opts: LocalStoreOptions): Promise { + const manifests: ProductManifest[] = []; + for (const key of await listProductKeys(opts)) { + const parsed = parseManifest(await readProductFile(key, 'manifest.json', opts)); + if (parsed) manifests.push(parsed); + } + return manifests; +} + +function parseManifest(raw: string | null): ProductManifest | undefined { + if (!raw) return undefined; + try { + const value = JSON.parse(raw) as ProductManifest; + if (value.schemaVersion !== 1 || !value.product?.key || !value.seed?.status) return undefined; + return value; + } catch { + return undefined; + } +} + +async function isLegacySite(productKey: string, opts: LocalStoreOptions): Promise { + const site = await readProductFile(productKey, 'sitemap/SITE.md', opts); + if (!site) return false; + if (parseManifest(await readProductFile(productKey, 'manifest.json', opts))) return /^---\s*\n/.test(site); + return true; +} + +async function listReferences(productKey: string, opts: LocalStoreOptions): Promise<{ path: string }[]> { + try { + const names = await readdir(join(sitesRoot(opts), productKey, 'sitemap', 'references')); + return names.filter((name) => name.endsWith('.md')).sort().map((name) => ({ path: `sitemap/references/${name}` })); + } catch { + return []; + } +} + +async function openGit(opts: LocalStoreOptions, diagnostics: string[]): Promise { + try { + return await openSitesRepository(opts); + } catch (err) { + diagnostics.push(err instanceof Error ? err.message : String(err)); + return null; + } +} + +function memorySegment(value: string): string { + if (!value || value.includes('/') || value.includes('\\') || value === '.' || value === '..' || value.startsWith('.')) { + throw new Error(`Invalid site memory path: ${value}`); + } + return value; +} diff --git a/src/site-memory/model.ts b/src/site-memory/model.ts index b5074e15..019e9a2f 100644 --- a/src/site-memory/model.ts +++ b/src/site-memory/model.ts @@ -16,10 +16,14 @@ export interface SeedPayload { references?: Record; } -export type SeedLookupResult = +export type PersistedSeedResult = | { status: 'unattempted' } | { status: 'absent' } | { status: 'lookup-failed' } + | { status: 'available'; revision: string }; + +export type SeedLookupResult = + | Exclude | ({ status: 'available' } & SeedPayload); export interface ProductManifest { @@ -27,7 +31,18 @@ export interface ProductManifest { product: ProductIdentity; /** Confirmed alternate hostnames belonging to this product. */ interfaces: ProductIdentity[]; - seed: SeedLookupResult; + seed: PersistedSeedResult; +} + +export interface MemoryContext { + resolution: ProductResolution; + manifest?: ProductManifest; + revision: MemoryRevision | null; + siteMarkdown: string | null; + references: { path: string }[]; + draftPath: string; + readOnly: boolean; + diagnostics: string[]; } export type ProductResolutionStatus = 'exact' | 'confirmed-interface' | 'provisional-fallback' | 'new'; diff --git a/src/site-memory/seed-client.test.ts b/src/site-memory/seed-client.test.ts new file mode 100644 index 00000000..97816a39 --- /dev/null +++ b/src/site-memory/seed-client.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it, vi } from 'vitest'; +import { createHttpSeedProvider } from './seed-client.js'; + +describe('global seed client', () => { + it('GETs the punycode seed URL without credentials and returns the JSON contract', async () => { + const fetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + expect(String(input)).toBe('https://api.webcmd.dev/v1/site-memory/seeds/xn--bcher-kva.example'); + expect(init?.method ?? 'GET').toBe('GET'); + expect(init?.credentials).toBe('omit'); + const headers = new Headers(init?.headers); + expect(headers.get('authorization')).toBeNull(); + return jsonResponse({ + revision: 'seed-1', + site: '# Bücher\n', + references: { 'old.md': '# Old\n' }, + }); + }); + + const result = await createHttpSeedProvider({ fetch, env: {} }).lookup('xn--bcher-kva.example'); + + expect(result).toEqual({ + status: 'available', + revision: 'seed-1', + site: '# Bücher\n', + references: { 'old.md': '# Old\n' }, + }); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it('uses WEBCMD_GLOBAL_MEMORY_URL as the request base', async () => { + const fetch = vi.fn(async (input: RequestInfo | URL) => { + expect(String(input)).toBe('https://memory.example/v1/site-memory/seeds/example.test'); + return jsonResponse({ revision: 'r1', site: '# Example\n' }); + }); + + await createHttpSeedProvider({ + fetch, + env: { WEBCMD_GLOBAL_MEMORY_URL: 'https://memory.example/' }, + }).lookup('example.test'); + + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it('treats HTTP 404 as absent', async () => { + const result = await createHttpSeedProvider({ + fetch: async () => new Response('missing', { status: 404 }), + env: {}, + }).lookup('missing.test'); + + expect(result).toEqual({ status: 'absent' }); + }); + + it('aborts after two seconds and does not retry', async () => { + let calls = 0; + const fetch = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => { + calls += 1; + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => { + reject(init.signal?.reason ?? new Error('aborted')); + }); + }); + }); + + const started = Date.now(); + const result = await createHttpSeedProvider({ fetch, env: {} }).lookup('slow.test'); + + expect(result).toEqual({ status: 'lookup-failed' }); + expect(calls).toBe(1); + expect(Date.now() - started).toBeGreaterThanOrEqual(2000); + expect(Date.now() - started).toBeLessThan(4000); + }); + + it('treats offline and malformed responses as lookup-failed', async () => { + const offline = await createHttpSeedProvider({ + fetch: async () => { + throw new TypeError('fetch failed'); + }, + env: {}, + }).lookup('offline.test'); + const malformed = await createHttpSeedProvider({ + fetch: async () => jsonResponse({ nope: true }), + env: {}, + }).lookup('bad.test'); + const unsafe = await createHttpSeedProvider({ + fetch: async () => jsonResponse({ + revision: 'r1', + site: '# Site\n', + references: { '../evil.md': '# no\n' }, + }), + env: {}, + }).lookup('evil.test'); + + expect(offline).toEqual({ status: 'lookup-failed' }); + expect(malformed).toEqual({ status: 'lookup-failed' }); + expect(unsafe).toEqual({ status: 'lookup-failed' }); + }); + + it('skips lookup when WEBCMD_GLOBAL_MEMORY=off', async () => { + const fetch = vi.fn(); + + const result = await createHttpSeedProvider({ + fetch, + env: { WEBCMD_GLOBAL_MEMORY: 'off' }, + }).lookup('example.test'); + + expect(result).toEqual({ status: 'unattempted' }); + expect(fetch).not.toHaveBeenCalled(); + }); +}); + +function jsonResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); +} diff --git a/src/site-memory/seed-client.ts b/src/site-memory/seed-client.ts new file mode 100644 index 00000000..3baaa80d --- /dev/null +++ b/src/site-memory/seed-client.ts @@ -0,0 +1,53 @@ +import type { SeedLookupResult } from './model.js'; + +export interface GlobalSeedProvider { + lookup(productKey: string, signal?: AbortSignal): Promise; +} + +export const DEFAULT_GLOBAL_MEMORY_URL = 'https://api.webcmd.dev'; +const LOOKUP_TIMEOUT_MS = 2000; + +export function createHttpSeedProvider(options: { + fetch?: typeof fetch; + env?: NodeJS.ProcessEnv; +} = {}): GlobalSeedProvider { + const fetchFn = options.fetch ?? fetch; + const env = options.env ?? process.env; + + return { + async lookup(productKey, signal) { + if (env.WEBCMD_GLOBAL_MEMORY === 'off') return { status: 'unattempted' }; + + const base = (env.WEBCMD_GLOBAL_MEMORY_URL ?? DEFAULT_GLOBAL_MEMORY_URL).replace(/\/+$/, ''); + const url = `${base}/v1/site-memory/seeds/${encodeURIComponent(productKey)}`; + const timeout = AbortSignal.timeout(LOOKUP_TIMEOUT_MS); + const combined = signal ? AbortSignal.any([signal, timeout]) : timeout; + try { + const response = await fetchFn(url, { method: 'GET', signal: combined, credentials: 'omit' }); + if (response.status === 404) return { status: 'absent' }; + if (!response.ok) return { status: 'lookup-failed' }; + return parseSeed(await response.json()); + } catch { + return { status: 'lookup-failed' }; + } + }, + }; +} + +function parseSeed(body: unknown): SeedLookupResult { + if (!body || typeof body !== 'object' || Array.isArray(body)) return { status: 'lookup-failed' }; + const { revision, site, references } = body as Record; + if (typeof revision !== 'string' || !revision || typeof site !== 'string') return { status: 'lookup-failed' }; + if (references === undefined) return { status: 'available', revision, site }; + if (!safeReferences(references)) return { status: 'lookup-failed' }; + return { status: 'available', revision, site, references }; +} + +function safeReferences(value: unknown): value is Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + return Object.entries(value).every(([key, body]) => typeof body === 'string' && safeReferenceName(key)); +} + +function safeReferenceName(key: string): boolean { + return Boolean(key) && !key.includes('/') && !key.includes('\\') && key !== '.' && key !== '..' && !key.startsWith('.'); +} From 0e6c325c61e88be141f948f3ace64384dc8e4178 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Mon, 31 Aug 2026 23:32:51 +0530 Subject: [PATCH 06/41] fix(site-memory): keep v1 memory writable and degrade persist failures --- src/site-memory/context.test.ts | 123 +++++++++++++++++++++++++++++++- src/site-memory/context.ts | 79 +++++++++++++++----- 2 files changed, 181 insertions(+), 21 deletions(-) diff --git a/src/site-memory/context.test.ts b/src/site-memory/context.test.ts index 7b96219b..06fe19cf 100644 --- a/src/site-memory/context.test.ts +++ b/src/site-memory/context.test.ts @@ -1,11 +1,12 @@ import { execFile } from 'node:child_process'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { getMemoryContext } from './context.js'; -import { readProductFile } from './local-store.js'; +import { openSitesRepository } from './git-store.js'; +import { readProductFile, writeProductFile } from './local-store.js'; import type { GlobalSeedProvider } from './seed-client.js'; import type { SeedLookupResult } from './model.js'; @@ -156,6 +157,124 @@ describe('memory context initialization', () => { seedProvider: provider(async () => ({ status: 'absent' })), })).rejects.toThrow(/Invalid site memory path/); }); + + it('does not treat schemaVersion 1 memory as beta because SITE.md has frontmatter', async () => { + const { homeDir } = await tempSites(); + await mkdir(join(homeDir, '.webcmd/sites/example.test/sitemap'), { recursive: true }); + await writeFile(join(homeDir, '.webcmd/sites/example.test/manifest.json'), `${JSON.stringify({ + schemaVersion: 1, + product: { + key: 'example.test', + hostname: 'example.test', + displayHostname: 'example.test', + registrableDomain: 'example.test', + }, + interfaces: [], + seed: { status: 'available', revision: 'seed-1' }, + }, null, 2)}\n`); + await writeFile( + join(homeDir, '.webcmd/sites/example.test/sitemap/SITE.md'), + '---\ntitle: seeded\n---\n# Seeded\n', + ); + const lookup = vi.fn(async () => ({ status: 'available' as const, revision: 'x', site: '# no\n' })); + + const context = await getMemoryContext({ + url: 'https://example.test/', + taskId: 'task-1', + homeDir, + seedProvider: provider(lookup), + }); + + expect(context.readOnly).toBe(false); + expect(context.diagnostics.join('\n')).not.toMatch(/incompatible beta schema/i); + expect(context.siteMarkdown).toBe('---\ntitle: seeded\n---\n# Seeded\n'); + expect(lookup).not.toHaveBeenCalled(); + }); + + it('degrades persist failures to read-only transient seed and drops uncommitted seed files', async () => { + const { homeDir } = await tempSites(); + await writeProductFile('other.test', 'manifest.json', '{}\n', { homeDir }); + await (await openSitesRepository({ homeDir })).commit(['other.test/manifest.json'], 'init'); + await writeProductFile('other.test', 'manifest.json', '{"dirty":true}\n', { homeDir }); + const lookup = vi.fn(async (): Promise => ({ + status: 'available', + revision: 'seed-1', + site: '# Transient\n', + references: { 'alt.md': '# Alt\n' }, + })); + + const context = await getMemoryContext({ + url: 'https://example.test/', + taskId: 'task-1', + homeDir, + seedProvider: provider(lookup), + }); + + expect(context.readOnly).toBe(true); + expect(context.siteMarkdown).toBe('# Transient\n'); + expect(context.manifest).toBeUndefined(); + expect(context.diagnostics.join('\n')).toMatch(/unrelated dirty path/i); + expect(await readProductFile('example.test', 'manifest.json', { homeDir })).toBeNull(); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBeNull(); + expect(await readProductFile('example.test', 'sitemap/references/alt.md', { homeDir })).toBeNull(); + expect(await readProductFile('other.test', 'manifest.json', { homeDir })).toBe('{"dirty":true}\n'); + + lookup.mockClear(); + await getMemoryContext({ + url: 'https://example.test/', + taskId: 'task-2', + homeDir, + seedProvider: provider(lookup), + }); + expect(lookup).toHaveBeenCalled(); + }); + + it('ignores manifests whose shape would crash product resolution', async () => { + const { homeDir } = await tempSites(); + await git(homeDir, ['init']); + const product = { + key: 'broken.test', + hostname: 'broken.test', + displayHostname: 'broken.test', + registrableDomain: 'broken.test', + }; + const lookup = vi.fn(async () => ({ status: 'absent' as const })); + for (const [key, manifest] of [ + ['missing-if.test', { schemaVersion: 1, product: { ...product, key: 'missing-if.test' }, seed: { status: 'absent' } }], + ['obj-if.test', { schemaVersion: 1, product: { ...product, key: 'obj-if.test' }, interfaces: {}, seed: { status: 'absent' } }], + ['null-if.test', { schemaVersion: 1, product: { ...product, key: 'null-if.test' }, interfaces: [null], seed: { status: 'absent' } }], + ['bad-product.test', { schemaVersion: 1, product: 'bad-product.test', interfaces: [], seed: { status: 'absent' } }], + ] as const) { + await mkdir(join(homeDir, '.webcmd/sites', key), { recursive: true }); + await writeFile(join(homeDir, '.webcmd/sites', key, 'manifest.json'), `${JSON.stringify(manifest)}\n`); + } + + for (const host of ['www.missing-if.test', 'www.obj-if.test', 'www.null-if.test', 'www.bad-product.test']) { + await expect(getMemoryContext({ + url: `https://${host}/`, + taskId: 'task-1', + homeDir, + seedProvider: provider(lookup), + })).resolves.toMatchObject({ resolution: { status: 'new' } }); + } + }); + + it.each(['absent', 'lookup-failed', 'unattempted'] as const)( + 'creates the task draft directory when seed is %s and SITE.md is absent', + async (status) => { + const { homeDir } = await tempSites(); + + const context = await getMemoryContext({ + url: 'https://example.test/', + taskId: 'task-1', + homeDir, + seedProvider: provider(async () => ({ status })), + }); + + expect(context.siteMarkdown).toBeNull(); + await expect(access(context.draftPath)).resolves.toBeUndefined(); + }, + ); }); function provider(lookup: GlobalSeedProvider['lookup']): GlobalSeedProvider { diff --git a/src/site-memory/context.ts b/src/site-memory/context.ts index e938fba2..3b30ecc7 100644 --- a/src/site-memory/context.ts +++ b/src/site-memory/context.ts @@ -1,4 +1,4 @@ -import { mkdir, readdir } from 'node:fs/promises'; +import { mkdir, readdir, unlink } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { openSitesRepository, type SitesRepository } from './git-store.js'; import { atomicWrite, containedRelativePath, listProductKeys, readProductFile, sitesRoot, writeProductFile } from './local-store.js'; @@ -28,12 +28,19 @@ export async function getMemoryContext(input: MemoryContextInput): Promise | undefined; if (resolution.status === 'new' && !legacy) { const seed = await (input.seedProvider ?? createHttpSeedProvider()).lookup(resolution.requested.key); if (git && seed.status !== 'unattempted') { - await persistSeed(resolution.requested, seed, git, opts); - resolution = resolveProduct(input.url, await loadManifests(opts)); + try { + await persistSeed(resolution.requested, seed, git, opts); + resolution = resolveProduct(input.url, await loadManifests(opts)); + } catch (err) { + persistFailed = true; + diagnostics.push(err instanceof Error ? err.message : String(err)); + if (seed.status === 'available') transient = seed; + } } else if (!git && seed.status === 'available') { transient = seed; } @@ -54,7 +61,7 @@ export async function getMemoryContext(input: MemoryContextInput): Promise { const persisted: PersistedSeedResult = seed.status === 'available' ? { status: 'available', revision: seed.revision } : seed; const manifest: ProductManifest = { schemaVersion: 1, product, interfaces: [], seed: persisted }; - const paths = [`${product.key}/manifest.json`]; - await writeProductFile(product.key, 'manifest.json', `${JSON.stringify(manifest, null, 2)}\n`, opts); - if (seed.status === 'available') { - await writeProductFile(product.key, 'sitemap/SITE.md', seed.site, opts); - paths.push(`${product.key}/sitemap/SITE.md`); - for (const [name, body] of Object.entries(seed.references ?? {})) { - const path = `sitemap/references/${name}`; - await writeProductFile(product.key, path, body, opts); - paths.push(`${product.key}/${path}`); + const files = ['manifest.json']; + try { + await writeProductFile(product.key, 'manifest.json', `${JSON.stringify(manifest, null, 2)}\n`, opts); + if (seed.status === 'available') { + await writeProductFile(product.key, 'sitemap/SITE.md', seed.site, opts); + files.push('sitemap/SITE.md'); + for (const [name, body] of Object.entries(seed.references ?? {})) { + const path = `sitemap/references/${name}`; + await writeProductFile(product.key, path, body, opts); + files.push(path); + } } + await repo.commit(files.map((file) => `${product.key}/${file}`), `initialize ${product.key}`); + } catch (err) { + await Promise.all(files.map((file) => unlinkProductFile(product.key, file, opts))); + throw err; } - await repo.commit(paths, `initialize ${product.key}`); +} + +async function unlinkProductFile(productKey: string, path: string, opts: LocalStoreOptions): Promise { + const productRoot = join(sitesRoot(opts), productKey); + const relative = containedRelativePath(productRoot, path); + await unlink(join(productRoot, ...relative.split('/'))).catch(() => undefined); } async function writeDraft( @@ -89,6 +107,7 @@ async function writeDraft( transientRefs: Record | undefined, opts: LocalStoreOptions, ): Promise { + await mkdir(draftPath, { recursive: true }); const draftRoot = join(draftPath, '..'); if (siteMarkdown != null) await writeContained(draftRoot, 'sitemap/SITE.md', siteMarkdown); for (const { path } of references) { @@ -116,19 +135,41 @@ async function loadManifests(opts: LocalStoreOptions): Promise; + return candidate.schemaVersion === 1 + && isProductIdentity(candidate.product) + && Array.isArray(candidate.interfaces) + && candidate.interfaces.every(isProductIdentity) + && isPersistedSeed(candidate.seed); +} + +function isProductIdentity(value: unknown): value is ProductIdentity { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const candidate = value as Record; + return [candidate.key, candidate.hostname, candidate.displayHostname, candidate.registrableDomain] + .every((field) => typeof field === 'string' && field.length > 0); +} + +function isPersistedSeed(value: unknown): value is PersistedSeedResult { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const candidate = value as Record; + if (candidate.status === 'unattempted' || candidate.status === 'absent' || candidate.status === 'lookup-failed') return true; + return candidate.status === 'available' && typeof candidate.revision === 'string' && candidate.revision.length > 0; +} + async function isLegacySite(productKey: string, opts: LocalStoreOptions): Promise { const site = await readProductFile(productKey, 'sitemap/SITE.md', opts); if (!site) return false; - if (parseManifest(await readProductFile(productKey, 'manifest.json', opts))) return /^---\s*\n/.test(site); - return true; + return !parseManifest(await readProductFile(productKey, 'manifest.json', opts)); } async function listReferences(productKey: string, opts: LocalStoreOptions): Promise<{ path: string }[]> { From 03bdb1ce43cd0ab63e6d01cde887e0030ffd1e1c Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Mon, 31 Aug 2026 23:40:42 +0530 Subject: [PATCH 07/41] fix(site-memory): serialize first-context seed writes under the repository lock Hold the repository lock across the post-acquire manifest re-check, seed writes, nested commit, and failure cleanup so a losing cold init cannot unlink the winner's committed files. Ignore ENOENT only during cleanup. --- src/site-memory/context.test.ts | 77 ++++++++++++++++++++++++++++++++- src/site-memory/context.ts | 43 ++++++++++-------- 2 files changed, 101 insertions(+), 19 deletions(-) diff --git a/src/site-memory/context.test.ts b/src/site-memory/context.test.ts index 06fe19cf..1bde5308 100644 --- a/src/site-memory/context.test.ts +++ b/src/site-memory/context.test.ts @@ -1,5 +1,5 @@ import { execFile } from 'node:child_process'; -import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { access, chmod, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; @@ -275,6 +275,81 @@ describe('memory context initialization', () => { await expect(access(context.draftPath)).resolves.toBeUndefined(); }, ); + + it('keeps one committed product when two cold contexts initialize together', async () => { + const { homeDir, sites } = await tempSites(); + const seed: SeedLookupResult = { + status: 'available', + revision: 'seed-1', + site: '# Seed\n', + references: { 'alt.md': '# Alt\n' }, + }; + let pending = 0; + let release!: () => void; + const bothLooking = new Promise((resolve) => { + release = resolve; + }); + const lookup = vi.fn(async (): Promise => { + pending += 1; + if (pending === 2) release(); + await bothLooking; + return seed; + }); + const input = { url: 'https://example.test/', homeDir, seedProvider: provider(lookup) }; + + const [first, second] = await Promise.all([ + getMemoryContext({ ...input, taskId: 'task-1' }), + getMemoryContext({ ...input, taskId: 'task-2' }), + ]); + + expect(await readProductFile('example.test', 'manifest.json', { homeDir })).not.toBeNull(); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBe('# Seed\n'); + expect(await readProductFile('example.test', 'sitemap/references/alt.md', { homeDir })).toBe('# Alt\n'); + expect(await git(sites, ['ls-files'])).toContain('example.test/manifest.json'); + expect(await git(sites, ['show', 'HEAD:example.test/sitemap/SITE.md'])).toBe('# Seed\n'); + for (const context of [first, second]) { + expect(context.resolution.status).toBe('exact'); + expect(context.manifest?.seed).toEqual({ status: 'available', revision: 'seed-1' }); + expect(context.siteMarkdown).toBe('# Seed\n'); + expect(context.readOnly).toBe(false); + } + }); + + it('surfaces non-ENOENT seed cleanup failures', async () => { + const { homeDir, sites } = await tempSites(); + const originalPath = process.env.PATH; + const productDir = join(sites, 'example.test'); + const wrapperDir = await mkdtemp(join(tmpdir(), 'webcmd-git-cleanup-')); + tempHomes.push(wrapperDir); + const { stdout } = await run('/usr/bin/which', ['git'], { encoding: 'utf8' }); + const wrapper = join(wrapperDir, 'git'); + await writeFile(wrapper, `#!/usr/bin/env node +const { chmodSync } = require('node:fs'); +const { spawnSync } = require('node:child_process'); +const args = process.argv.slice(2); +if (args.includes('commit')) { + try { chmodSync(${JSON.stringify(productDir)}, 0o555); } catch {} + process.exit(1); +} +const result = spawnSync(${JSON.stringify(stdout.trim())}, args, { stdio: 'inherit' }); +process.exit(result.status ?? 1); +`); + await chmod(wrapper, 0o755); + process.env.PATH = `${wrapperDir}:${originalPath}`; + try { + const context = await getMemoryContext({ + url: 'https://example.test/', + taskId: 'task-1', + homeDir, + seedProvider: provider(async () => ({ status: 'available', revision: 'seed-1', site: '# Transient\n' })), + }); + expect(context.readOnly).toBe(true); + expect(context.diagnostics.join('\n')).toMatch(/EACCES|EPERM|permission denied/i); + } finally { + process.env.PATH = originalPath; + await chmod(productDir, 0o755).catch(() => undefined); + } + }); }); function provider(lookup: GlobalSeedProvider['lookup']): GlobalSeedProvider { diff --git a/src/site-memory/context.ts b/src/site-memory/context.ts index 3b30ecc7..362de94d 100644 --- a/src/site-memory/context.ts +++ b/src/site-memory/context.ts @@ -72,31 +72,38 @@ async function persistSeed( repo: SitesRepository, opts: LocalStoreOptions, ): Promise { - const persisted: PersistedSeedResult = seed.status === 'available' ? { status: 'available', revision: seed.revision } : seed; - const manifest: ProductManifest = { schemaVersion: 1, product, interfaces: [], seed: persisted }; - const files = ['manifest.json']; - try { - await writeProductFile(product.key, 'manifest.json', `${JSON.stringify(manifest, null, 2)}\n`, opts); - if (seed.status === 'available') { - await writeProductFile(product.key, 'sitemap/SITE.md', seed.site, opts); - files.push('sitemap/SITE.md'); - for (const [name, body] of Object.entries(seed.references ?? {})) { - const path = `sitemap/references/${name}`; - await writeProductFile(product.key, path, body, opts); - files.push(path); + await repo.withRepositoryLock(async () => { + if (parseManifest(await readProductFile(product.key, 'manifest.json', opts))) return; + const persisted: PersistedSeedResult = seed.status === 'available' ? { status: 'available', revision: seed.revision } : seed; + const manifest: ProductManifest = { schemaVersion: 1, product, interfaces: [], seed: persisted }; + const files = ['manifest.json']; + try { + await writeProductFile(product.key, 'manifest.json', `${JSON.stringify(manifest, null, 2)}\n`, opts); + if (seed.status === 'available') { + await writeProductFile(product.key, 'sitemap/SITE.md', seed.site, opts); + files.push('sitemap/SITE.md'); + for (const [name, body] of Object.entries(seed.references ?? {})) { + const path = `sitemap/references/${name}`; + await writeProductFile(product.key, path, body, opts); + files.push(path); + } } + await repo.commit(files.map((file) => `${product.key}/${file}`), `initialize ${product.key}`); + } catch (err) { + await Promise.all(files.map((file) => unlinkProductFile(product.key, file, opts))); + throw err; } - await repo.commit(files.map((file) => `${product.key}/${file}`), `initialize ${product.key}`); - } catch (err) { - await Promise.all(files.map((file) => unlinkProductFile(product.key, file, opts))); - throw err; - } + }); } async function unlinkProductFile(productKey: string, path: string, opts: LocalStoreOptions): Promise { const productRoot = join(sitesRoot(opts), productKey); const relative = containedRelativePath(productRoot, path); - await unlink(join(productRoot, ...relative.split('/'))).catch(() => undefined); + try { + await unlink(join(productRoot, ...relative.split('/'))); + } catch (err) { + if (!(err instanceof Error) || !('code' in err) || err.code !== 'ENOENT') throw err; + } } async function writeDraft( From d5131b6d596d23ec3cb928f60ca7a3dcb6ed8320 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Mon, 31 Aug 2026 23:55:33 +0530 Subject: [PATCH 08/41] feat(site-memory): retain selective candidate evidence --- src/site-memory/candidates.test.ts | 172 ++++++++++++++++++++++++++++ src/site-memory/candidates.ts | 159 +++++++++++++++++++++++++ src/site-memory/environment.test.ts | 71 ++++++++++++ src/site-memory/environment.ts | 62 ++++++++++ src/site-memory/local-store.ts | 9 +- src/site-memory/model.ts | 15 +++ 6 files changed, 486 insertions(+), 2 deletions(-) create mode 100644 src/site-memory/candidates.test.ts create mode 100644 src/site-memory/candidates.ts create mode 100644 src/site-memory/environment.test.ts create mode 100644 src/site-memory/environment.ts diff --git a/src/site-memory/candidates.test.ts b/src/site-memory/candidates.test.ts new file mode 100644 index 00000000..3850f3e8 --- /dev/null +++ b/src/site-memory/candidates.test.ts @@ -0,0 +1,172 @@ +import { execFile } from 'node:child_process'; +import { mkdtemp, readdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import { afterEach, describe, expect, it } from 'vitest'; +import { addCandidate, listCandidates, searchCandidates, showCandidate } from './candidates.js'; +import { listSiteMemory, showSiteMemory, writeProductFile } from './local-store.js'; +import type { Candidate } from './model.js'; + +const run = promisify(execFile); +const tempHomes: string[] = []; + +afterEach(async () => { + await Promise.all(tempHomes.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +describe('candidate capture', () => { + it('writes unique lexical filenames and derives UTC date from offset timestamps', async () => { + const { homeDir, sites } = await tempSites(); + const first = await addCandidate(base(homeDir, { + claim: 'Old Reddit is denser', + observedAt: '2026-08-31T00:30:00+05:30', + })); + const second = await addCandidate(base(homeDir, { + claim: 'RSS skips paging', + observedAt: '2026-08-31T00:30:01+05:30', + })); + + const files = (await readdir(join(sites, 'example.test/candidates'))).sort(); + expect(first.id).not.toBe(second.id); + expect(first.observedDateUtc).toBe('2026-08-30'); + expect(files).toEqual([`${first.id}.json`, `${second.id}.json`]); + expect(files[0] < files[1]).toBe(true); + expect(files[0]).toMatch(/^20260830T190000Z-/); + expect(files[1]).toMatch(/^20260830T190001Z-/); + }); + + it('rejects invalid schema, paths, kinds, and secret-bearing fields', async () => { + const { homeDir } = await tempSites(); + + await expect(addCandidate(base(homeDir, { kind: 'trivial_success' }))).rejects.toThrow(/kind/i); + await expect(addCandidate(base(homeDir, { product: '../escape' }))).rejects.toThrow(/invalid/i); + await expect(addCandidate(base(homeDir, { claim: 'Cookie: session=abc' }))).rejects.toThrow(/secret/i); + await expect(addCandidate({ + ...base(homeDir), + password: 'hunter2', + } as Parameters[0] & { password: string })).rejects.toThrow(/secret/i); + await expect(showCandidate('example.test', '../manifest.json', { homeDir })).rejects.toThrow(/invalid/i); + }); + + it('captures even when provenance collection fails', async () => { + const { homeDir } = await tempSites(); + + const summary = await addCandidate(base(homeDir, { + environment: undefined, + fetch: async () => { + throw new TypeError('offline'); + }, + })); + const stored = await showCandidate('example.test', summary.id, { homeDir }); + + expect(summary.status).toBe('pending'); + expect(stored.evidence).toBe('Used /new while /hot spun.'); + expect(stored.environment.publicIp).toBeUndefined(); + }); + + it('commits each candidate once and keeps concurrent captures', async () => { + const { homeDir, sites } = await tempSites(); + const [a, b] = await Promise.all([ + addCandidate(base(homeDir, { claim: 'First concurrent path' })), + addCandidate(base(homeDir, { claim: 'Second concurrent path' })), + ]); + + const files = (await git(sites, ['ls-files', '--', 'example.test/candidates'])).trim().split('\n').sort(); + expect(files).toEqual([`example.test/candidates/${a.id}.json`, `example.test/candidates/${b.id}.json`].sort()); + expect((await git(sites, ['log', '--oneline', '--', `example.test/candidates/${a.id}.json`])).trim().split('\n')).toHaveLength(1); + expect((await git(sites, ['log', '--oneline', '--', `example.test/candidates/${b.id}.json`])).trim().split('\n')).toHaveLength(1); + expect((await git(sites, ['log', '--oneline'])).trim().split('\n')).toHaveLength(2); + }); +}); + +describe('candidate discovery', () => { + it('searches pending candidates with compact lexical ranking and hides completed ones', async () => { + const { homeDir } = await tempSites(); + const ranked = await addCandidate(base(homeDir, { + hostname: 'old.reddit.com', + kind: 'better_path', + claim: 'Old Reddit denser listing', + consequence: 'Fewer page loads', + })); + const weaker = await addCandidate(base(homeDir, { + hostname: 'www.reddit.com', + kind: 'access', + claim: 'Login wall on old posts', + consequence: 'Need an account', + })); + const ingested = await addCandidate(base(homeDir, { + hostname: 'old.reddit.com', + kind: 'better_path', + claim: 'Old Reddit denser listing again', + consequence: 'Fewer page loads', + })); + await markStatus(homeDir, ingested.id, 'ingested'); + + const hits = await searchCandidates('example.test', 'old reddit denser', 10, { homeDir }); + + expect(hits.map((hit) => hit.id)).toEqual([ranked.id, weaker.id]); + expect(hits[0]).toEqual(expect.objectContaining({ + id: ranked.id, + kind: 'better_path', + hostname: 'old.reddit.com', + claim: 'Old Reddit denser listing', + consequence: 'Fewer page loads', + status: 'pending', + })); + expect(hits[0]).not.toHaveProperty('evidence'); + expect(hits[0]).not.toHaveProperty('environment'); + expect(await searchCandidates('example.test', 'old reddit denser', 1, { homeDir })).toEqual([hits[0]]); + expect((await listCandidates('example.test', { homeDir })).map((item) => item.id).sort()).toEqual( + [ranked.id, weaker.id, ingested.id].sort(), + ); + }); + + it('hides candidates and raw environment values from ordinary memory listing', async () => { + const { homeDir } = await tempSites(); + await writeProductFile('example.test', 'notes.md', 'hello\n', { homeDir }); + const summary = await addCandidate(base(homeDir, { + environment: { publicIp: '203.0.113.9', localIp: '192.168.1.8', machine: 'secret-host' }, + })); + + const listed = await listSiteMemory('example.test', { homeDir }); + const shown = await showSiteMemory('example.test', { homeDir }); + const explicit = await listSiteMemory('example.test', { homeDir, paths: [`candidates/${summary.id}.json`] }); + + expect(listed.map((item) => item.path)).toEqual(['notes.md']); + expect(shown.map((item) => item.path)).toEqual(['notes.md']); + expect(explicit).toEqual([]); + expect(JSON.stringify({ listed, shown, explicit })).not.toMatch(/203\.0\.113\.9|192\.168\.1\.8|secret-host/); + expect((await showCandidate('example.test', summary.id, { homeDir })).environment.publicIp).toBe('203.0.113.9'); + }); +}); + +function base(homeDir: string, extra: Record = {}) { + return { + product: 'example.test', + hostname: 'www.example.test', + kind: 'better_path', + claim: 'New listing is faster', + evidence: 'Used /new while /hot spun.', + consequence: 'Prefer /new for fresh posts', + environment: {}, + homeDir, + ...extra, + }; +} + +async function markStatus(homeDir: string, id: string, status: Candidate['status']) { + const stored = await showCandidate('example.test', id, { homeDir }); + await writeProductFile('example.test', `candidates/${id}.json`, `${JSON.stringify({ ...stored, status }, null, 2)}\n`, { homeDir }); +} + +async function tempSites() { + const homeDir = await mkdtemp(join(tmpdir(), 'webcmd-candidates-')); + tempHomes.push(homeDir); + return { homeDir, sites: join(homeDir, '.webcmd', 'sites') }; +} + +async function git(cwd: string, args: string[]) { + const { stdout } = await run('git', args, { cwd, encoding: 'utf8' }); + return stdout; +} diff --git a/src/site-memory/candidates.ts b/src/site-memory/candidates.ts new file mode 100644 index 00000000..ea310cd2 --- /dev/null +++ b/src/site-memory/candidates.ts @@ -0,0 +1,159 @@ +import { randomUUID } from 'node:crypto'; +import { readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { collectEnvironment } from './environment.js'; +import { openSitesRepository } from './git-store.js'; +import { readProductFile, sitesRoot, writeProductFile, type LocalStoreOptions } from './local-store.js'; +import { CANDIDATE_KINDS, type Candidate, type CandidateSummary } from './model.js'; +import { canonicalProductKey } from './product-resolver.js'; + +const SECRET_KEY = /^(password|passwd|secret|token|cookie|cookies|authorization|api[_-]?key|set-cookie)$/i; +const SECRET_TEXT = /(password\s*[:=]|secret\s*[:=]|api[_-]?key|authorization\s*:|bearer\s+\S+|cookie\s*[:=])/i; +const KINDS = new Set(CANDIDATE_KINDS); + +export interface AddCandidateInput extends LocalStoreOptions { + product: string; + hostname?: string; + kind: string; + claim: string; + evidence: string; + consequence: string; + observedAt?: string; + environment?: Candidate['environment']; + browserVersion?: string; + webcmdVersion?: string; + fetch?: typeof fetch; +} + +export async function addCandidate(input: AddCandidateInput): Promise { + rejectSecrets(input); + if (!KINDS.has(input.kind)) throw new Error(`Invalid candidate kind: ${input.kind}`); + const product = canonicalProductKey(input.product); + const host = canonicalProductKey(input.hostname ?? input.product); + const observedAt = input.observedAt ?? new Date().toISOString(); + const observed = new Date(observedAt); + if (Number.isNaN(observed.getTime())) throw new Error(`Invalid observedAt: ${observedAt}`); + const id = `${compactUtc(observed)}-${randomUUID()}`; + const candidate: Candidate = { + schemaVersion: 1, + id, + domain: host.registrableDomain, + hostname: host.hostname, + observedAt, + observedDateUtc: observed.toISOString().slice(0, 10), + kind: input.kind, + claim: requiredText(input.claim, 'claim'), + evidence: requiredText(input.evidence, 'evidence'), + consequence: requiredText(input.consequence, 'consequence'), + environment: input.environment ?? await collectEnvironment({ + browserVersion: input.browserVersion, + webcmdVersion: input.webcmdVersion, + fetch: input.fetch, + }), + status: 'pending', + evidenceRole: null, + memoryCommit: null, + reviewedAt: null, + rejectionReason: null, + }; + const relative = candidatePath(id); + await writeProductFile(product.key, relative, `${JSON.stringify(candidate, null, 2)}\n`, input); + const repo = await openSitesRepository(input); + await repo.commit([`${product.key}/${relative}`], `capture candidate ${id}`); + return toSummary(candidate); +} + +export async function searchCandidates( + product: string, + query: string, + limit = 20, + opts: LocalStoreOptions = {}, +): Promise { + const tokens = query.toLowerCase().split(/\s+/).filter(Boolean); + const ranked = (await loadCandidates(product, opts)) + .filter((candidate) => candidate.status === 'pending') + .map((candidate) => ({ candidate, score: fieldMatches(candidate, tokens) })) + .filter((entry) => tokens.length === 0 || entry.score > 0) + .sort((a, b) => b.score - a.score || a.candidate.observedAt.localeCompare(b.candidate.observedAt) || a.candidate.id.localeCompare(b.candidate.id)); + return ranked.slice(0, limit).map((entry) => toSummary(entry.candidate)); +} + +export async function listCandidates(product: string, opts: LocalStoreOptions = {}): Promise { + return (await loadCandidates(product, opts)).map(toSummary); +} + +export async function showCandidate(product: string, id: string, opts: LocalStoreOptions = {}): Promise { + const key = canonicalProductKey(product).key; + const body = await readProductFile(key, candidatePath(id), opts); + if (body === null) throw new Error(`Candidate ${id} was not found.`); + return JSON.parse(body) as Candidate; +} + +function candidatePath(id: string): string { + if (!id || id.includes('/') || id.includes('\\') || id === '.' || id === '..' || id.startsWith('.')) { + throw new Error(`Invalid site memory path: ${id}`); + } + return `candidates/${id}.json`; +} + +async function loadCandidates(product: string, opts: LocalStoreOptions): Promise { + const key = canonicalProductKey(product).key; + const dir = join(sitesRoot(opts), key, 'candidates'); + let names: string[]; + try { + names = (await readdir(dir)).filter((name) => name.endsWith('.json')).sort(); + } catch (err) { + if (isEnoent(err)) return []; + throw err; + } + const loaded = await Promise.all(names.map(async (name) => { + const body = await readProductFile(key, `candidates/${name}`, opts); + return body ? JSON.parse(body) as Candidate : null; + })); + return loaded.filter((candidate): candidate is Candidate => candidate !== null); +} + +function fieldMatches(candidate: Candidate, tokens: string[]): number { + const fields = [candidate.claim, candidate.kind, candidate.hostname, candidate.consequence].map((value) => value.toLowerCase()); + let score = 0; + for (const token of tokens) { + for (const field of fields) { + if (field.includes(token)) score += 1; + } + } + return score; +} + +function toSummary(candidate: Candidate): CandidateSummary { + return { + id: candidate.id, + domain: candidate.domain, + hostname: candidate.hostname, + observedAt: candidate.observedAt, + observedDateUtc: candidate.observedDateUtc, + kind: candidate.kind, + claim: candidate.claim, + consequence: candidate.consequence, + status: candidate.status, + }; +} + +function rejectSecrets(input: object): void { + for (const key of Object.keys(input)) { + if (SECRET_KEY.test(key)) throw new Error('Candidate evidence cannot include secret-bearing fields.'); + } +} + +function requiredText(value: string, field: string): string { + if (!value.trim()) throw new Error(`Invalid candidate ${field}.`); + if (SECRET_TEXT.test(value)) throw new Error('Candidate evidence cannot include secret-bearing fields.'); + return value; +} + +function compactUtc(date: Date): string { + return date.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}Z$/, 'Z'); +} + +function isEnoent(err: unknown): boolean { + return err instanceof Error && 'code' in err && err.code === 'ENOENT'; +} diff --git a/src/site-memory/environment.test.ts b/src/site-memory/environment.test.ts new file mode 100644 index 00000000..73d2f7fc --- /dev/null +++ b/src/site-memory/environment.test.ts @@ -0,0 +1,71 @@ +import * as os from 'node:os'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { collectEnvironment } from './environment.js'; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('candidate environment provenance', () => { + it('includes independently requested fields and omits the rest', async () => { + const env = await collectEnvironment({ + machine: false, + localIp: false, + publicIp: false, + os: false, + browserVersion: '1.61.1', + webcmdVersion: '0.7.11', + }); + + expect(env).toEqual({ browserVersion: '1.61.1', webcmdVersion: '0.7.11' }); + }); + + it('collects local provenance from Node OS APIs', async () => { + const env = await collectEnvironment({ publicIp: false, browserVersion: false, webcmdVersion: false }); + const addresses = Object.values(os.networkInterfaces()).flat().filter((entry) => entry && !entry.internal && entry.family === 'IPv4'); + + expect(env.machine).toEqual(expect.any(String)); + expect(env.os).toEqual(expect.any(String)); + if (addresses[0]) expect(env.localIp).toBe(addresses[0].address); + }); + + it('omits fields when OS or public-IP lookup fails', async () => { + const env = await collectEnvironment({ + localIp: false, + os: false, + browserVersion: false, + webcmdVersion: false, + hostname: () => { + throw new Error('no hostname'); + }, + fetch: async () => { + throw new TypeError('fetch failed'); + }, + }); + + expect(env).toEqual({}); + }); + + it('bounds public-IP lookup and omits the field on timeout', async () => { + const fetch = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => { + reject(init.signal?.reason ?? new Error('aborted')); + }); + })); + + const started = Date.now(); + const env = await collectEnvironment({ + machine: false, + localIp: false, + os: false, + browserVersion: false, + webcmdVersion: false, + fetch, + }); + + expect(env.publicIp).toBeUndefined(); + expect(fetch).toHaveBeenCalledTimes(1); + expect(Date.now() - started).toBeGreaterThanOrEqual(2000); + expect(Date.now() - started).toBeLessThan(4000); + }); +}); diff --git a/src/site-memory/environment.ts b/src/site-memory/environment.ts new file mode 100644 index 00000000..3c92298d --- /dev/null +++ b/src/site-memory/environment.ts @@ -0,0 +1,62 @@ +import { isIP } from 'node:net'; +import * as os from 'node:os'; +import { PKG_VERSION } from '../version.js'; +import type { CandidateEnvironment } from './model.js'; + +const PUBLIC_IP_TIMEOUT_MS = 2000; +const PUBLIC_IP_URL = 'https://api.ipify.org'; + +export interface CollectEnvironmentOptions { + machine?: boolean; + localIp?: boolean; + publicIp?: boolean; + os?: boolean; + browserVersion?: string | false; + webcmdVersion?: string | false; + fetch?: typeof fetch; + hostname?: () => string; +} + +export async function collectEnvironment(options: CollectEnvironmentOptions = {}): Promise { + const env: CandidateEnvironment = {}; + if (options.machine !== false) { + try { env.machine = (options.hostname ?? os.hostname)(); } catch {} + } + if (options.localIp !== false) { + try { + const localIp = firstLocalIp(); + if (localIp) env.localIp = localIp; + } catch {} + } + if (options.os !== false) { + try { env.os = `${os.type()} ${os.release()}`; } catch {} + } + if (typeof options.browserVersion === 'string') env.browserVersion = options.browserVersion; + if (options.webcmdVersion !== false) { + env.webcmdVersion = typeof options.webcmdVersion === 'string' ? options.webcmdVersion : PKG_VERSION; + } + if (options.publicIp !== false) { + const publicIp = await lookupPublicIp(options.fetch ?? fetch); + if (publicIp) env.publicIp = publicIp; + } + return env; +} + +function firstLocalIp(): string | undefined { + for (const entries of Object.values(os.networkInterfaces())) { + for (const entry of entries ?? []) { + if (!entry.internal && entry.family === 'IPv4') return entry.address; + } + } +} + +async function lookupPublicIp(fetchFn: typeof fetch): Promise { + try { + const response = await fetchFn(PUBLIC_IP_URL, { signal: AbortSignal.timeout(PUBLIC_IP_TIMEOUT_MS) }); + if (!response.ok) return; + const value = (await response.text()).trim(); + return isIP(value) ? value : undefined; + } catch { + return undefined; + } +} diff --git a/src/site-memory/local-store.ts b/src/site-memory/local-store.ts index 134b2fe8..322977a6 100644 --- a/src/site-memory/local-store.ts +++ b/src/site-memory/local-store.ts @@ -296,7 +296,7 @@ function productSegment(value: string): string { async function memoryPaths(root: string, requested?: string[]): Promise { if (!await exists(root)) return []; - const paths = (requested ?? await walkFiles(root)).filter((path) => !isInternalWritePath(path)); + const paths = (requested ?? await walkFiles(root)).filter((path) => !isInternalWritePath(path) && !isCandidateMemoryPath(path)); return Promise.all(paths.map((path) => readableRelativePath(root, path))).then((items) => items.sort()); } @@ -304,7 +304,7 @@ async function walkFiles(root: string, dir = root): Promise { const entries = await readdir(dir, { withFileTypes: true }); const nested = await Promise.all(entries.map(async (entry) => { const path = join(dir, entry.name); - if (entry.isDirectory()) return walkFiles(root, path); + if (entry.isDirectory()) return entry.name === 'candidates' ? [] : walkFiles(root, path); if (entry.isFile() && !isInternalWritePath(entry.name)) return [relative(root, path)]; return []; })); @@ -332,6 +332,11 @@ function isInternalWritePath(path: string): boolean { return tempWritePattern.test(name) || lockWritePattern.test(name); } +function isCandidateMemoryPath(path: string): boolean { + const normalized = path.split(sep).join('/'); + return normalized === 'candidates' || normalized.startsWith('candidates/'); +} + function requiredHomeDir(opts: LocalStoreOptions): string { const home = opts.homeDir ?? process.env.HOME ?? process.env.USERPROFILE ?? homedir(); if (!home) throw new Error('Site memory requires homeDir, HOME, USERPROFILE, or os.homedir().'); diff --git a/src/site-memory/model.ts b/src/site-memory/model.ts index 019e9a2f..cef658fa 100644 --- a/src/site-memory/model.ts +++ b/src/site-memory/model.ts @@ -58,6 +58,9 @@ export interface ProductResolution { export type CandidateStatus = 'pending' | 'ingested' | 'rejected'; +export const CANDIDATE_KINDS = ['action_space', 'better_path', 'access', 'high_consequence', 'repeated_mistake'] as const; +export type CandidateKind = (typeof CANDIDATE_KINDS)[number]; + export interface CandidateEnvironment { machine?: string; localIp?: string; @@ -85,3 +88,15 @@ export interface Candidate { reviewedAt: string | null; rejectionReason: string | null; } + +export interface CandidateSummary { + id: string; + domain: string; + hostname: string; + observedAt: string; + observedDateUtc: string; + kind: string; + claim: string; + consequence: string; + status: CandidateStatus; +} From 972820fc4afcb60f812f93d360b4fbac31a86e7f Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Tue, 1 Sep 2026 00:05:40 +0530 Subject: [PATCH 09/41] fix(site-memory): encode candidates with the approved schema Write snake_case candidate JSON through an explicit codec, reject untrusted loads, serialize capture write+commit under the repository lock with own-file cleanup, and hard-cap search at 20. --- src/site-memory/candidates.test.ts | 171 ++++++++++++++++++++++++- src/site-memory/candidates.ts | 196 ++++++++++++++++++++++++++--- 2 files changed, 349 insertions(+), 18 deletions(-) diff --git a/src/site-memory/candidates.test.ts b/src/site-memory/candidates.test.ts index 3850f3e8..7ac35664 100644 --- a/src/site-memory/candidates.test.ts +++ b/src/site-memory/candidates.test.ts @@ -1,11 +1,11 @@ import { execFile } from 'node:child_process'; -import { mkdtemp, readdir, rm } from 'node:fs/promises'; +import { chmod, mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; import { afterEach, describe, expect, it } from 'vitest'; import { addCandidate, listCandidates, searchCandidates, showCandidate } from './candidates.js'; -import { listSiteMemory, showSiteMemory, writeProductFile } from './local-store.js'; +import { listSiteMemory, readProductFile, showSiteMemory, writeProductFile } from './local-store.js'; import type { Candidate } from './model.js'; const run = promisify(execFile); @@ -139,6 +139,156 @@ describe('candidate discovery', () => { expect(JSON.stringify({ listed, shown, explicit })).not.toMatch(/203\.0\.113\.9|192\.168\.1\.8|secret-host/); expect((await showCandidate('example.test', summary.id, { homeDir })).environment.publicIp).toBe('203.0.113.9'); }); + + it('persists the approved snake_case candidate schema', async () => { + const { homeDir } = await tempSites(); + const summary = await addCandidate(base(homeDir, { + observedAt: '2026-08-31T14:23:00+05:30', + environment: { + machine: 'box', + localIp: '192.168.1.8', + publicIp: '203.0.113.9', + os: 'Darwin 24.0', + browserVersion: '1.61.1', + webcmdVersion: '0.7.11', + }, + })); + + const raw = JSON.parse(await readProductFile('example.test', `candidates/${summary.id}.json`, { homeDir }) ?? ''); + expect(raw).toEqual({ + schema_version: 1, + id: summary.id, + domain: 'example.test', + hostname: 'www.example.test', + observed_at: '2026-08-31T14:23:00+05:30', + observed_date_utc: '2026-08-31', + kind: 'better_path', + claim: 'New listing is faster', + evidence: 'Used /new while /hot spun.', + consequence: 'Prefer /new for fresh posts', + environment: { + machine: 'box', + local_ip: '192.168.1.8', + public_ip: '203.0.113.9', + os: 'Darwin 24.0', + browser_version: '1.61.1', + webcmd_version: '0.7.11', + }, + status: 'pending', + evidence_role: null, + memory_commit: null, + reviewed_at: null, + rejection_reason: null, + }); + }); + + it('fails closed on malformed, unknown, secret, and mismatched candidate JSON', async () => { + const { homeDir } = await tempSites(); + const summary = await addCandidate(base(homeDir)); + const path = `candidates/${summary.id}.json`; + const raw = JSON.parse(await readProductFile('example.test', path, { homeDir }) ?? ''); + + await writeProductFile('example.test', 'candidates/not-json.json', '{\n', { homeDir }); + await expect(showCandidate('example.test', 'not-json', { homeDir })).rejects.toThrow(/invalid|json|parse/i); + await expect(listCandidates('example.test', { homeDir })).rejects.toThrow(/invalid|json|parse/i); + await expect(searchCandidates('example.test', 'listing', 10, { homeDir })).rejects.toThrow(/invalid|json|parse/i); + + await writeProductFile('example.test', 'candidates/not-json.json', `${JSON.stringify({ ...raw, extra: true }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', 'not-json', { homeDir })).rejects.toThrow(/invalid/i); + + await writeProductFile('example.test', path, `${JSON.stringify({ ...raw, schema_version: 2 }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/schema/i); + + await writeProductFile('example.test', path, `${JSON.stringify({ ...raw, status: 'draft' }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/status/i); + + await writeProductFile('example.test', path, `${JSON.stringify({ ...raw, evidence_role: 'maybe' }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/role/i); + + await writeProductFile('example.test', path, `${JSON.stringify({ ...raw, kind: 'trivial_success' }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/kind/i); + + await writeProductFile('example.test', path, `${JSON.stringify({ ...raw, cookie: 'session' }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/secret/i); + + await writeProductFile('example.test', `candidates/other-id.json`, `${JSON.stringify(raw, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', 'other-id', { homeDir })).rejects.toThrow(/mismatch|invalid/i); + + await expect(addCandidate(base(homeDir, { environment: { cookie: 'session' } }))).rejects.toThrow(/secret/i); + await expect(addCandidate(base(homeDir, { environment: { extra: 'nope' } }))).rejects.toThrow(/environment|invalid/i); + await expect(addCandidate(base(homeDir, { environment: { localIp: { nested: true } } }))).rejects.toThrow(/environment|invalid/i); + }); + + it('does not leave an uncommitted candidate when git open or commit fails', async () => { + const ancestor = await tempSites(); + await mkdir(ancestor.sites, { recursive: true }); + await git(ancestor.homeDir, ['init']); + await expect(addCandidate(base(ancestor.homeDir))).rejects.toThrow(/ancestor/i); + expect(await jsonNames(ancestor.sites)).toEqual([]); + + const { homeDir, sites } = await tempSites(); + const kept = await addCandidate(base(homeDir, { claim: 'Keep concurrent unique capture' })); + await writeProductFile('example.test', `candidates/${kept.id}.json`, 'dirty\n', { homeDir }); + await expect(addCandidate(base(homeDir, { claim: 'Transient uncommitted row' }))).rejects.toThrow(/unrelated/i); + expect(await jsonNames(sites)).toEqual([`${kept.id}.json`]); + }); + + it('reports non-ENOENT cleanup errors after a failed commit', async () => { + const { homeDir, sites } = await tempSites(); + const originalPath = process.env.PATH; + const candidatesDir = join(sites, 'example.test', 'candidates'); + const wrapperDir = await mkdtemp(join(tmpdir(), 'webcmd-candidate-cleanup-')); + tempHomes.push(wrapperDir); + const { stdout } = await run('/usr/bin/which', ['git'], { encoding: 'utf8' }); + const wrapper = join(wrapperDir, 'git'); + await writeFile(wrapper, `#!/usr/bin/env node +const { chmodSync } = require('node:fs'); +const { spawnSync } = require('node:child_process'); +const args = process.argv.slice(2); +if (args.includes('commit')) { + try { chmodSync(${JSON.stringify(candidatesDir)}, 0o555); } catch {} + process.exit(1); +} +const result = spawnSync(${JSON.stringify(stdout.trim())}, args, { stdio: 'inherit' }); +process.exit(result.status ?? 1); +`); + await chmod(wrapper, 0o755); + process.env.PATH = `${wrapperDir}:${originalPath}`; + try { + await expect(addCandidate(base(homeDir))).rejects.toThrow(/EACCES|EPERM|permission denied/i); + } finally { + process.env.PATH = originalPath; + await chmod(candidatesDir, 0o755).catch(() => undefined); + } + }); + + it('validates and hard-caps search limit with deterministic ordering', async () => { + const { homeDir } = await tempSites(); + await expect(searchCandidates('example.test', 'listing', 0, { homeDir })).rejects.toThrow(/limit/i); + await expect(searchCandidates('example.test', 'listing', -1, { homeDir })).rejects.toThrow(/limit/i); + await expect(searchCandidates('example.test', 'listing', 1.5, { homeDir })).rejects.toThrow(/limit/i); + + const earlier = await addCandidate(base(homeDir, { + claim: 'Equal score path', + observedAt: '2026-08-31T01:00:00Z', + })); + const later = await addCandidate(base(homeDir, { + claim: 'Equal score path', + observedAt: '2026-08-31T02:00:00Z', + })); + const ordered = await searchCandidates('example.test', 'equal score path', 10, { homeDir }); + expect(ordered.map((hit) => hit.id)).toEqual([earlier.id, later.id]); + + await Promise.all(Array.from({ length: 21 }, (_, index) => addCandidate(base(homeDir, { + claim: `Cap row ${index}`, + observedAt: `2026-08-31T03:00:${String(index).padStart(2, '0')}Z`, + })))); + const capped = await searchCandidates('example.test', 'cap row', 999, { homeDir }); + expect(capped).toHaveLength(20); + expect(capped.map((hit) => hit.id)).toEqual([...capped].sort((a, b) => ( + a.observedAt.localeCompare(b.observedAt) || a.id.localeCompare(b.id) + )).map((hit) => hit.id).slice(0, 20)); + }, 20_000); }); function base(homeDir: string, extra: Record = {}) { @@ -156,8 +306,21 @@ function base(homeDir: string, extra: Record = {}) { } async function markStatus(homeDir: string, id: string, status: Candidate['status']) { - const stored = await showCandidate('example.test', id, { homeDir }); - await writeProductFile('example.test', `candidates/${id}.json`, `${JSON.stringify({ ...stored, status }, null, 2)}\n`, { homeDir }); + const path = `candidates/${id}.json`; + const body = await readProductFile('example.test', path, { homeDir }); + if (body === null) throw new Error(`missing ${id}`); + const raw = JSON.parse(body) as { status: string }; + raw.status = status; + await writeProductFile('example.test', path, `${JSON.stringify(raw, null, 2)}\n`, { homeDir }); +} + +async function jsonNames(sites: string): Promise { + try { + return (await readdir(join(sites, 'example.test', 'candidates'))).filter((name) => name.endsWith('.json')).sort(); + } catch (err) { + if (err instanceof Error && 'code' in err && err.code === 'ENOENT') return []; + throw err; + } } async function tempSites() { diff --git a/src/site-memory/candidates.ts b/src/site-memory/candidates.ts index ea310cd2..0143ce34 100644 --- a/src/site-memory/candidates.ts +++ b/src/site-memory/candidates.ts @@ -1,15 +1,31 @@ import { randomUUID } from 'node:crypto'; -import { readdir } from 'node:fs/promises'; +import { readdir, unlink } from 'node:fs/promises'; import { join } from 'node:path'; import { collectEnvironment } from './environment.js'; import { openSitesRepository } from './git-store.js'; -import { readProductFile, sitesRoot, writeProductFile, type LocalStoreOptions } from './local-store.js'; -import { CANDIDATE_KINDS, type Candidate, type CandidateSummary } from './model.js'; +import { containedRelativePath, readProductFile, sitesRoot, writeProductFile, type LocalStoreOptions } from './local-store.js'; +import { + CANDIDATE_KINDS, + type Candidate, + type CandidateEnvironment, + type CandidateStatus, + type CandidateSummary, +} from './model.js'; import { canonicalProductKey } from './product-resolver.js'; +export const SEARCH_CANDIDATE_LIMIT = 20; + const SECRET_KEY = /^(password|passwd|secret|token|cookie|cookies|authorization|api[_-]?key|set-cookie)$/i; const SECRET_TEXT = /(password\s*[:=]|secret\s*[:=]|api[_-]?key|authorization\s*:|bearer\s+\S+|cookie\s*[:=])/i; const KINDS = new Set(CANDIDATE_KINDS); +const STATUSES = new Set(['pending', 'ingested', 'rejected']); +const CANDIDATE_FIELDS = new Set([ + 'schema_version', 'id', 'domain', 'hostname', 'observed_at', 'observed_date_utc', + 'kind', 'claim', 'evidence', 'consequence', 'environment', 'status', + 'evidence_role', 'memory_commit', 'reviewed_at', 'rejection_reason', +]); +const ENV_FIELDS = new Set(['machine', 'local_ip', 'public_ip', 'os', 'browser_version', 'webcmd_version']); +const CALLER_ENV_FIELDS = new Set(['machine', 'localIp', 'publicIp', 'os', 'browserVersion', 'webcmdVersion']); export interface AddCandidateInput extends LocalStoreOptions { product: string; @@ -33,6 +49,13 @@ export async function addCandidate(input: AddCandidateInput): Promise { + try { + await writeProductFile(product.key, relative, `${JSON.stringify(encodeCandidate(candidate), null, 2)}\n`, input); + await repo.commit([`${product.key}/${relative}`], `capture candidate ${id}`); + } catch (err) { + await unlinkProductFile(product.key, relative, input); + throw err; + } + }); return toSummary(candidate); } export async function searchCandidates( product: string, query: string, - limit = 20, + limit = SEARCH_CANDIDATE_LIMIT, opts: LocalStoreOptions = {}, ): Promise { + const cap = boundedSearchLimit(limit); const tokens = query.toLowerCase().split(/\s+/).filter(Boolean); const ranked = (await loadCandidates(product, opts)) .filter((candidate) => candidate.status === 'pending') .map((candidate) => ({ candidate, score: fieldMatches(candidate, tokens) })) .filter((entry) => tokens.length === 0 || entry.score > 0) .sort((a, b) => b.score - a.score || a.candidate.observedAt.localeCompare(b.candidate.observedAt) || a.candidate.id.localeCompare(b.candidate.id)); - return ranked.slice(0, limit).map((entry) => toSummary(entry.candidate)); + return ranked.slice(0, cap).map((entry) => toSummary(entry.candidate)); } export async function listCandidates(product: string, opts: LocalStoreOptions = {}): Promise { @@ -86,7 +113,7 @@ export async function showCandidate(product: string, id: string, opts: LocalStor const key = canonicalProductKey(product).key; const body = await readProductFile(key, candidatePath(id), opts); if (body === null) throw new Error(`Candidate ${id} was not found.`); - return JSON.parse(body) as Candidate; + return parseCandidate(body, id); } function candidatePath(id: string): string { @@ -108,11 +135,142 @@ async function loadCandidates(product: string, opts: LocalStoreOptions): Promise } const loaded = await Promise.all(names.map(async (name) => { const body = await readProductFile(key, `candidates/${name}`, opts); - return body ? JSON.parse(body) as Candidate : null; + return body ? parseCandidate(body, name.slice(0, -'.json'.length)) : null; })); return loaded.filter((candidate): candidate is Candidate => candidate !== null); } +function encodeCandidate(candidate: Candidate): Record { + return { + schema_version: candidate.schemaVersion, + id: candidate.id, + domain: candidate.domain, + hostname: candidate.hostname, + observed_at: candidate.observedAt, + observed_date_utc: candidate.observedDateUtc, + kind: candidate.kind, + claim: candidate.claim, + evidence: candidate.evidence, + consequence: candidate.consequence, + environment: encodeEnvironment(candidate.environment), + status: candidate.status, + evidence_role: candidate.evidenceRole, + memory_commit: candidate.memoryCommit, + reviewed_at: candidate.reviewedAt, + rejection_reason: candidate.rejectionReason, + }; +} + +function encodeEnvironment(env: CandidateEnvironment): Record { + const encoded: Record = {}; + if (env.machine !== undefined) encoded.machine = env.machine; + if (env.localIp !== undefined) encoded.local_ip = env.localIp; + if (env.publicIp !== undefined) encoded.public_ip = env.publicIp; + if (env.os !== undefined) encoded.os = env.os; + if (env.browserVersion !== undefined) encoded.browser_version = env.browserVersion; + if (env.webcmdVersion !== undefined) encoded.webcmd_version = env.webcmdVersion; + return encoded; +} + +function parseCandidate(body: string, expectedId: string): Candidate { + let value: unknown; + try { + value = JSON.parse(body); + } catch { + throw new Error('Invalid candidate JSON.'); + } + return decodeCandidate(value, expectedId); +} + +function decodeCandidate(value: unknown, expectedId: string): Candidate { + const raw = knownObject(value, CANDIDATE_FIELDS, 'JSON'); + const id = requiredString(raw.id, 'id'); + if (id !== expectedId) throw new Error('Invalid candidate id mismatch.'); + const kind = requiredString(raw.kind, 'kind'); + if (!KINDS.has(kind)) throw new Error(`Invalid candidate kind: ${kind}`); + const status = requiredString(raw.status, 'status'); + if (!STATUSES.has(status as CandidateStatus)) throw new Error(`Invalid candidate status: ${status}`); + if (raw.schema_version !== 1) throw new Error('Invalid candidate schema_version.'); + const evidenceRole = raw.evidence_role === null || raw.evidence_role === 'supporting' || raw.evidence_role === 'dissenting' + ? raw.evidence_role + : null; + if (raw.evidence_role !== evidenceRole) throw new Error('Invalid candidate evidence_role.'); + return { + schemaVersion: 1, + id, + domain: requiredString(raw.domain, 'domain'), + hostname: requiredString(raw.hostname, 'hostname'), + observedAt: requiredString(raw.observed_at, 'observed_at'), + observedDateUtc: requiredString(raw.observed_date_utc, 'observed_date_utc'), + kind, + claim: requiredText(requiredString(raw.claim, 'claim'), 'claim'), + evidence: requiredText(requiredString(raw.evidence, 'evidence'), 'evidence'), + consequence: requiredText(requiredString(raw.consequence, 'consequence'), 'consequence'), + environment: decodeEnvironment(raw.environment), + status: status as CandidateStatus, + evidenceRole, + memoryCommit: nullableString(raw.memory_commit, 'memory_commit'), + reviewedAt: nullableString(raw.reviewed_at, 'reviewed_at'), + rejectionReason: nullableString(raw.rejection_reason, 'rejection_reason'), + }; +} + +function decodeEnvironment(value: unknown): CandidateEnvironment { + const raw = knownObject(value, ENV_FIELDS, 'environment'); + return { + ...optionalString(raw, 'machine', 'machine'), + ...optionalString(raw, 'local_ip', 'localIp'), + ...optionalString(raw, 'public_ip', 'publicIp'), + ...optionalString(raw, 'os', 'os'), + ...optionalString(raw, 'browser_version', 'browserVersion'), + ...optionalString(raw, 'webcmd_version', 'webcmdVersion'), + }; +} + +function decodeCallerEnvironment(value: unknown): CandidateEnvironment { + const raw = knownObject(value, CALLER_ENV_FIELDS, 'environment'); + return { + ...optionalString(raw, 'machine', 'machine'), + ...optionalString(raw, 'localIp', 'localIp'), + ...optionalString(raw, 'publicIp', 'publicIp'), + ...optionalString(raw, 'os', 'os'), + ...optionalString(raw, 'browserVersion', 'browserVersion'), + ...optionalString(raw, 'webcmdVersion', 'webcmdVersion'), + }; +} + +function knownObject(value: unknown, allowed: Set, label: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`Invalid candidate ${label}.`); + const obj = value as Record; + for (const key of Object.keys(obj)) { + if (SECRET_KEY.test(key)) throw new Error('Candidate evidence cannot include secret-bearing fields.'); + if (!allowed.has(key)) throw new Error(`Invalid candidate ${label}.`); + } + return obj; +} + +function optionalString(obj: Record, from: string, to: keyof CandidateEnvironment): CandidateEnvironment { + if (obj[from] === undefined) return {}; + if (typeof obj[from] !== 'string' || !obj[from]) throw new Error('Invalid candidate environment.'); + return { [to]: obj[from] }; +} + +function requiredString(value: unknown, field: string): string { + if (typeof value !== 'string' || !value) throw new Error(`Invalid candidate ${field}.`); + return value; +} + +function nullableString(value: unknown, field: string): string | null { + if (value === null) return null; + if (typeof value !== 'string' || !value) throw new Error(`Invalid candidate ${field}.`); + return value; +} + +function boundedSearchLimit(limit: number): number { + if (!Number.isInteger(limit) || limit < 1) throw new Error(`Invalid search limit: ${limit}`); + return Math.min(limit, SEARCH_CANDIDATE_LIMIT); +} + function fieldMatches(candidate: Candidate, tokens: string[]): number { const fields = [candidate.claim, candidate.kind, candidate.hostname, candidate.consequence].map((value) => value.toLowerCase()); let score = 0; @@ -154,6 +312,16 @@ function compactUtc(date: Date): string { return date.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}Z$/, 'Z'); } +async function unlinkProductFile(productKey: string, path: string, opts: LocalStoreOptions): Promise { + const productRoot = join(sitesRoot(opts), productKey); + const relative = containedRelativePath(productRoot, path); + try { + await unlink(join(productRoot, ...relative.split('/'))); + } catch (err) { + if (!isEnoent(err)) throw err; + } +} + function isEnoent(err: unknown): boolean { return err instanceof Error && 'code' in err && err.code === 'ENOENT'; } From 5fe82f3ecfc434310af5e7316668903554ea6bd9 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Tue, 1 Sep 2026 00:13:58 +0530 Subject: [PATCH 10/41] fix(site-memory): restore staged paths after a failed commit --- src/site-memory/candidates.test.ts | 112 ++++++++++++++++++++++++++++- src/site-memory/candidates.ts | 70 +++++++++++++++--- src/site-memory/git-store.test.ts | 56 ++++++++++++++- src/site-memory/git-store.ts | 20 +++++- 4 files changed, 244 insertions(+), 14 deletions(-) diff --git a/src/site-memory/candidates.test.ts b/src/site-memory/candidates.test.ts index 7ac35664..a60ba6a3 100644 --- a/src/site-memory/candidates.test.ts +++ b/src/site-memory/candidates.test.ts @@ -217,6 +217,105 @@ describe('candidate discovery', () => { await expect(addCandidate(base(homeDir, { environment: { cookie: 'session' } }))).rejects.toThrow(/secret/i); await expect(addCandidate(base(homeDir, { environment: { extra: 'nope' } }))).rejects.toThrow(/environment|invalid/i); await expect(addCandidate(base(homeDir, { environment: { localIp: { nested: true } } }))).rejects.toThrow(/environment|invalid/i); + await expect(addCandidate(base(homeDir, { environment: { machine: 'password: hunter2' } }))).rejects.toThrow(/secret/i); + }); + + it('rejects invalid timestamps, hosts, environment secrets, and status metadata', async () => { + const { homeDir } = await tempSites(); + const summary = await addCandidate(base(homeDir, { observedAt: '2026-08-31T14:23:00+05:30' })); + const path = `candidates/${summary.id}.json`; + const raw = JSON.parse(await readProductFile('example.test', path, { homeDir }) ?? ''); + + await writeProductFile('example.test', path, `${JSON.stringify({ ...raw, observed_at: 'not-a-timestamp' }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/observed_at|timestamp|invalid/i); + + await writeProductFile('example.test', path, `${JSON.stringify({ ...raw, observed_date_utc: '2026-08-30' }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/observed_date_utc|invalid/i); + + await writeProductFile('example.test', path, `${JSON.stringify({ ...raw, hostname: 'not a host' }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/hostname|invalid/i); + + await writeProductFile('example.test', path, `${JSON.stringify({ ...raw, domain: 'www.example.test' }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/domain|invalid/i); + + await writeProductFile('example.test', path, `${JSON.stringify({ + ...raw, + environment: { ...raw.environment, machine: 'password: hunter2' }, + }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/secret/i); + + await writeProductFile('example.test', path, `${JSON.stringify({ ...raw, evidence_role: 'supporting' }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/status|role|invalid/i); + + await writeProductFile('example.test', path, `${JSON.stringify({ + ...raw, + status: 'ingested', + evidence_role: 'supporting', + memory_commit: 'abc', + reviewed_at: '2026-08-31T14:23:00Z', + rejection_reason: 'nope', + }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/status|rejection|invalid/i); + + await writeProductFile('example.test', path, `${JSON.stringify({ + ...raw, + status: 'ingested', + evidence_role: null, + memory_commit: 'abc', + reviewed_at: '2026-08-31T14:23:00Z', + rejection_reason: null, + }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/status|role|invalid/i); + + await writeProductFile('example.test', path, `${JSON.stringify({ + ...raw, + status: 'rejected', + evidence_role: null, + memory_commit: null, + reviewed_at: null, + rejection_reason: 'transient', + }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/status|reviewed|invalid/i); + + await writeProductFile('example.test', path, `${JSON.stringify({ + ...raw, + status: 'rejected', + evidence_role: 'dissenting', + memory_commit: null, + reviewed_at: '2026-08-31T14:23:00Z', + rejection_reason: 'transient', + }, null, 2)}\n`, { homeDir }); + await expect(showCandidate('example.test', summary.id, { homeDir })).rejects.toThrow(/status|role|invalid/i); + }); + + it('does not leave a staged candidate after commit failure', async () => { + const { homeDir, sites } = await tempSites(); + await mkdir(sites, { recursive: true }); + await writeFile(join(sites, 'keep-me.txt'), 'unrelated\n'); + const originalPath = process.env.PATH; + const wrapperDir = await mkdtemp(join(tmpdir(), 'webcmd-candidate-commit-fail-')); + tempHomes.push(wrapperDir); + const { stdout } = await run('/usr/bin/which', ['git'], { encoding: 'utf8' }); + const wrapper = join(wrapperDir, 'git'); + await writeFile(wrapper, `#!/usr/bin/env node +const { spawnSync } = require('node:child_process'); +const args = process.argv.slice(2); +if (args.includes('commit')) process.exit(1); +const result = spawnSync(${JSON.stringify(stdout.trim())}, args, { stdio: 'inherit' }); +process.exit(result.status ?? 1); +`); + await chmod(wrapper, 0o755); + process.env.PATH = `${wrapperDir}:${originalPath}`; + try { + await expect(addCandidate(base(homeDir))).rejects.toThrow(); + } finally { + process.env.PATH = originalPath; + } + + expect(await jsonNames(sites)).toEqual([]); + expect((await git(sites, ['ls-files', '--stage'])).trim()).toBe(''); + expect(await git(sites, ['status', '--porcelain', '-uall'])).toMatch(/^\?\? keep-me\.txt$/m); + expect(await git(sites, ['status', '--porcelain', '-uall'])).not.toMatch(/candidates/); }); it('does not leave an uncommitted candidate when git open or commit fails', async () => { @@ -309,8 +408,19 @@ async function markStatus(homeDir: string, id: string, status: Candidate['status const path = `candidates/${id}.json`; const body = await readProductFile('example.test', path, { homeDir }); if (body === null) throw new Error(`missing ${id}`); - const raw = JSON.parse(body) as { status: string }; + const raw = JSON.parse(body) as Record; raw.status = status; + if (status === 'ingested') { + raw.evidence_role = 'supporting'; + raw.memory_commit = 'abc'; + raw.reviewed_at = '2026-08-31T14:23:00Z'; + raw.rejection_reason = null; + } else if (status === 'rejected') { + raw.evidence_role = null; + raw.memory_commit = null; + raw.reviewed_at = '2026-08-31T14:23:00Z'; + raw.rejection_reason = 'transient'; + } await writeProductFile('example.test', path, `${JSON.stringify(raw, null, 2)}\n`, { homeDir }); } diff --git a/src/site-memory/candidates.ts b/src/site-memory/candidates.ts index 0143ce34..2c9d38cc 100644 --- a/src/site-memory/candidates.ts +++ b/src/site-memory/candidates.ts @@ -195,13 +195,24 @@ function decodeCandidate(value: unknown, expectedId: string): Candidate { ? raw.evidence_role : null; if (raw.evidence_role !== evidenceRole) throw new Error('Invalid candidate evidence_role.'); + const observedAt = requiredString(raw.observed_at, 'observed_at'); + const observed = new Date(observedAt); + if (Number.isNaN(observed.getTime())) throw new Error('Invalid candidate observed_at.'); + const observedDateUtc = requiredString(raw.observed_date_utc, 'observed_date_utc'); + if (observedDateUtc !== observed.toISOString().slice(0, 10)) throw new Error('Invalid candidate observed_date_utc.'); + const hostname = requiredCanonicalHost(raw.hostname, 'hostname'); + const domain = requiredCanonicalHost(raw.domain, 'domain'); + const memoryCommit = nullableString(raw.memory_commit, 'memory_commit'); + const reviewedAt = nullableString(raw.reviewed_at, 'reviewed_at'); + const rejectionReason = nullableString(raw.rejection_reason, 'rejection_reason'); + assertStatusMetadata(status as CandidateStatus, evidenceRole, memoryCommit, reviewedAt, rejectionReason); return { schemaVersion: 1, id, - domain: requiredString(raw.domain, 'domain'), - hostname: requiredString(raw.hostname, 'hostname'), - observedAt: requiredString(raw.observed_at, 'observed_at'), - observedDateUtc: requiredString(raw.observed_date_utc, 'observed_date_utc'), + domain, + hostname, + observedAt, + observedDateUtc, kind, claim: requiredText(requiredString(raw.claim, 'claim'), 'claim'), evidence: requiredText(requiredString(raw.evidence, 'evidence'), 'evidence'), @@ -209,12 +220,49 @@ function decodeCandidate(value: unknown, expectedId: string): Candidate { environment: decodeEnvironment(raw.environment), status: status as CandidateStatus, evidenceRole, - memoryCommit: nullableString(raw.memory_commit, 'memory_commit'), - reviewedAt: nullableString(raw.reviewed_at, 'reviewed_at'), - rejectionReason: nullableString(raw.rejection_reason, 'rejection_reason'), + memoryCommit, + reviewedAt, + rejectionReason, }; } +function requiredCanonicalHost(value: unknown, field: 'hostname' | 'domain'): string { + const text = requiredString(value, field); + let identity; + try { + identity = canonicalProductKey(text); + } catch { + throw new Error(`Invalid candidate ${field}.`); + } + const expected = field === 'domain' ? identity.registrableDomain : identity.hostname; + if (expected !== text) throw new Error(`Invalid candidate ${field}.`); + return text; +} + +function assertStatusMetadata( + status: CandidateStatus, + evidenceRole: 'supporting' | 'dissenting' | null, + memoryCommit: string | null, + reviewedAt: string | null, + rejectionReason: string | null, +): void { + if (status === 'pending') { + if (evidenceRole !== null || memoryCommit !== null || reviewedAt !== null || rejectionReason !== null) { + throw new Error('Invalid candidate status.'); + } + return; + } + if (status === 'ingested') { + if ((evidenceRole !== 'supporting' && evidenceRole !== 'dissenting') || memoryCommit === null || reviewedAt === null || rejectionReason !== null) { + throw new Error('Invalid candidate status.'); + } + return; + } + if (evidenceRole !== null || memoryCommit !== null || reviewedAt === null || rejectionReason === null) { + throw new Error('Invalid candidate status.'); + } +} + function decodeEnvironment(value: unknown): CandidateEnvironment { const raw = knownObject(value, ENV_FIELDS, 'environment'); return { @@ -250,9 +298,11 @@ function knownObject(value: unknown, allowed: Set, label: string): Recor } function optionalString(obj: Record, from: string, to: keyof CandidateEnvironment): CandidateEnvironment { - if (obj[from] === undefined) return {}; - if (typeof obj[from] !== 'string' || !obj[from]) throw new Error('Invalid candidate environment.'); - return { [to]: obj[from] }; + const value = obj[from]; + if (value === undefined) return {}; + if (typeof value !== 'string' || !value) throw new Error('Invalid candidate environment.'); + if (SECRET_TEXT.test(value)) throw new Error('Candidate evidence cannot include secret-bearing fields.'); + return { [to]: value }; } function requiredString(value: unknown, field: string): string { diff --git a/src/site-memory/git-store.test.ts b/src/site-memory/git-store.test.ts index e36e069e..4e5f7a83 100644 --- a/src/site-memory/git-store.test.ts +++ b/src/site-memory/git-store.test.ts @@ -1,5 +1,5 @@ import { execFile, spawnSync } from 'node:child_process'; -import { chmod, mkdir, mkdtemp, realpath, rm, utimes, writeFile } from 'node:fs/promises'; +import { chmod, mkdir, mkdtemp, readFile, realpath, rm, utimes, writeFile } from 'node:fs/promises'; import { hostname, tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; @@ -152,6 +152,44 @@ describe('sites git repository', () => { await expect(repo.withRepositoryLock(async () => 'ok')).resolves.toBe('ok'); }); + it('restores only staged paths after a failed first commit without HEAD', async () => { + const { homeDir, sites } = await tempSites(); + await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); + await mkdir(join(sites, '.drafts', 'task'), { recursive: true }); + await writeFile(join(sites, '.drafts', 'task', 'scratch.md'), 'draft'); + await writeFile(join(sites, 'keep-me.txt'), 'unrelated\n'); + await installFailingCommitGit(); + + await expect( + (await openSitesRepository({ homeDir })).commit(['example.test/manifest.json'], 'init'), + ).rejects.toThrow(); + + expect((await git(sites, ['ls-files'])).trim()).toBe(''); + expect(await git(sites, ['status', '--porcelain', '-uall'])).toMatch(/^\?\? keep-me\.txt$/m); + expect(await git(sites, ['status', '--porcelain', '-uall'])).not.toMatch(/^(A |AD|D )/m); + expect(await readFile(join(sites, 'example.test', 'manifest.json'), 'utf8')).toBe('{}\n'); + await expect(git(sites, ['rev-parse', 'HEAD'])).rejects.toThrow(); + }); + + it('restores only staged paths after a failed commit when HEAD exists', async () => { + const { homeDir, sites } = await tempSites(); + await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); + const repo = await openSitesRepository({ homeDir }); + await repo.commit(['example.test/manifest.json'], 'init'); + const head = (await git(sites, ['rev-parse', 'HEAD'])).trim(); + await writeProductFile('example.test', 'notes.md', 'keep later\n', { homeDir }); + await writeFile(join(sites, 'keep-me.txt'), 'unrelated\n'); + await installFailingCommitGit(); + + await expect(repo.commit(['example.test/notes.md'], 'notes')).rejects.toThrow(); + + expect((await git(sites, ['ls-files'])).trim().split('\n').sort()).toEqual(['.gitignore', 'example.test/manifest.json']); + expect((await git(sites, ['rev-parse', 'HEAD'])).trim()).toBe(head); + expect(await git(sites, ['status', '--porcelain', '-uall'])).toMatch(/^\?\? keep-me\.txt$/m); + expect(await git(sites, ['status', '--porcelain', '-uall'])).toMatch(/^\?\? example\.test\/notes\.md$/m); + expect(await git(sites, ['status', '--porcelain', '-uall'])).not.toMatch(/^(A |AD)/m); + }); + it('excludes .git, .drafts, and other dot entries from product enumeration', async () => { const { homeDir, sites } = await tempSites(); await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); @@ -175,6 +213,22 @@ async function git(cwd: string, args: string[]) { return stdout; } +async function installFailingCommitGit() { + const dir = await mkdtemp(join(tmpdir(), 'webcmd-fail-git-')); + tempHomes.push(dir); + const { stdout } = await run('/usr/bin/which', ['git'], { encoding: 'utf8' }); + const wrapper = join(dir, 'git'); + await writeFile(wrapper, `#!/usr/bin/env node +const { spawnSync } = require('node:child_process'); +const args = process.argv.slice(2); +if (args.includes('commit')) process.exit(1); +const result = spawnSync(${JSON.stringify(stdout.trim())}, args, { stdio: 'inherit' }); +process.exit(result.status ?? 1); +`); + await chmod(wrapper, 0o755); + process.env.PATH = `${dir}:${process.env.PATH}`; +} + async function installSlowGit(delayMs: number) { const dir = await mkdtemp(join(tmpdir(), 'webcmd-slow-git-')); tempHomes.push(dir); diff --git a/src/site-memory/git-store.ts b/src/site-memory/git-store.ts index 5cb4930f..bf65df0d 100644 --- a/src/site-memory/git-store.ts +++ b/src/site-memory/git-store.ts @@ -52,11 +52,27 @@ async function commitPaths(root: string, paths: string[], message: string): Prom const relativePaths = paths.map((path) => containedRelativePath(root, path)); await atomicWrite(join(root, '.gitignore'), GITIGNORE); await assertNoUnrelatedDirty(root, relativePaths); - await git(root, ['add', '--', ...relativePaths, '.gitignore']); - await git(root, ['commit', '--no-gpg-sign', '-m', message]); + try { + await git(root, ['add', '--', ...relativePaths, '.gitignore']); + await git(root, ['commit', '--no-gpg-sign', '-m', message]); + } catch (err) { + await restoreStagedPaths(root, relativePaths); + throw err; + } return (await git(root, ['rev-parse', 'HEAD'])).trim(); } +async function restoreStagedPaths(root: string, relativePaths: string[]): Promise { + const paths = [...relativePaths, '.gitignore']; + try { + if (await revisionOf(root) !== null) { + await git(root, ['restore', '--staged', '--', ...paths]); + } else { + await git(root, ['rm', '--cached', '-f', '--ignore-unmatch', '--', ...paths]); + } + } catch {} +} + async function ensureRepository(root: string): Promise { if (!await hasExactRepository(root)) { await git(root, ['init']); From d612d5cdf9c42fc55d6232ba475cb0ae2b7d1e2d Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Tue, 1 Sep 2026 00:22:52 +0530 Subject: [PATCH 11/41] fix(site-memory): surface cleanup failure with failed commit If index restore also fails after a commit error, throw AggregateError with both causes instead of swallowing cleanup. --- src/site-memory/git-store.test.ts | 40 +++++++++++++++++++++++++++++++ src/site-memory/git-store.ts | 18 +++++++------- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/src/site-memory/git-store.test.ts b/src/site-memory/git-store.test.ts index 4e5f7a83..faa1f40f 100644 --- a/src/site-memory/git-store.test.ts +++ b/src/site-memory/git-store.test.ts @@ -190,6 +190,30 @@ describe('sites git repository', () => { expect(await git(sites, ['status', '--porcelain', '-uall'])).not.toMatch(/^(A |AD)/m); }); + it('exposes commit and cleanup failures together when restore also fails', async () => { + const { homeDir } = await tempSites(); + await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); + const repo = await openSitesRepository({ homeDir }); + await repo.commit(['example.test/manifest.json'], 'init'); + await writeProductFile('example.test', 'notes.md', 'keep later\n', { homeDir }); + await installFailingCommitAndCleanupGit(); + + const error = await repo.commit(['example.test/notes.md'], 'notes').then( + () => { + throw new Error('expected commit to fail'); + }, + (err: unknown) => err, + ); + + expect(error).toBeInstanceOf(AggregateError); + const aggregate = error as AggregateError; + expect(aggregate.errors).toHaveLength(2); + expect(aggregate.message).toMatch(/cleanup/i); + expect(aggregate.errors.map((item) => (item instanceof Error ? item.message : String(item))).join('\n')).toMatch( + /restore|rm --cached/, + ); + }); + it('excludes .git, .drafts, and other dot entries from product enumeration', async () => { const { homeDir, sites } = await tempSites(); await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); @@ -229,6 +253,22 @@ process.exit(result.status ?? 1); process.env.PATH = `${dir}:${process.env.PATH}`; } +async function installFailingCommitAndCleanupGit() { + const dir = await mkdtemp(join(tmpdir(), 'webcmd-fail-git-cleanup-')); + tempHomes.push(dir); + const { stdout } = await run('/usr/bin/which', ['git'], { encoding: 'utf8' }); + const wrapper = join(dir, 'git'); + await writeFile(wrapper, `#!/usr/bin/env node +const { spawnSync } = require('node:child_process'); +const args = process.argv.slice(2); +if (args.includes('commit') || args.includes('restore') || args.includes('rm')) process.exit(1); +const result = spawnSync(${JSON.stringify(stdout.trim())}, args, { stdio: 'inherit' }); +process.exit(result.status ?? 1); +`); + await chmod(wrapper, 0o755); + process.env.PATH = `${dir}:${process.env.PATH}`; +} + async function installSlowGit(delayMs: number) { const dir = await mkdtemp(join(tmpdir(), 'webcmd-slow-git-')); tempHomes.push(dir); diff --git a/src/site-memory/git-store.ts b/src/site-memory/git-store.ts index bf65df0d..2944e8a1 100644 --- a/src/site-memory/git-store.ts +++ b/src/site-memory/git-store.ts @@ -56,7 +56,11 @@ async function commitPaths(root: string, paths: string[], message: string): Prom await git(root, ['add', '--', ...relativePaths, '.gitignore']); await git(root, ['commit', '--no-gpg-sign', '-m', message]); } catch (err) { - await restoreStagedPaths(root, relativePaths); + try { + await restoreStagedPaths(root, relativePaths); + } catch (cleanupErr) { + throw new AggregateError([err, cleanupErr], 'Commit failed and index cleanup also failed'); + } throw err; } return (await git(root, ['rev-parse', 'HEAD'])).trim(); @@ -64,13 +68,11 @@ async function commitPaths(root: string, paths: string[], message: string): Prom async function restoreStagedPaths(root: string, relativePaths: string[]): Promise { const paths = [...relativePaths, '.gitignore']; - try { - if (await revisionOf(root) !== null) { - await git(root, ['restore', '--staged', '--', ...paths]); - } else { - await git(root, ['rm', '--cached', '-f', '--ignore-unmatch', '--', ...paths]); - } - } catch {} + if (await revisionOf(root) !== null) { + await git(root, ['restore', '--staged', '--', ...paths]); + } else { + await git(root, ['rm', '--cached', '-f', '--ignore-unmatch', '--', ...paths]); + } } async function ensureRepository(root: string): Promise { From a12b632c3b705cffbe0836ba12508bca4277c717 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Tue, 1 Sep 2026 00:37:48 +0530 Subject: [PATCH 12/41] feat(site-memory): checkpoint learned memory safely --- src/site-memory/candidates.ts | 9 + src/site-memory/checkpoint.test.ts | 446 +++++++++++++++++++++++++++++ src/site-memory/checkpoint.ts | 246 ++++++++++++++++ src/site-memory/model.ts | 14 + 4 files changed, 715 insertions(+) create mode 100644 src/site-memory/checkpoint.test.ts create mode 100644 src/site-memory/checkpoint.ts diff --git a/src/site-memory/candidates.ts b/src/site-memory/candidates.ts index 2c9d38cc..4b9a2bcd 100644 --- a/src/site-memory/candidates.ts +++ b/src/site-memory/candidates.ts @@ -110,12 +110,21 @@ export async function listCandidates(product: string, opts: LocalStoreOptions = } export async function showCandidate(product: string, id: string, opts: LocalStoreOptions = {}): Promise { + return readCandidateRecord(product, id, opts); +} + +export async function readCandidateRecord(product: string, id: string, opts: LocalStoreOptions = {}): Promise { const key = canonicalProductKey(product).key; const body = await readProductFile(key, candidatePath(id), opts); if (body === null) throw new Error(`Candidate ${id} was not found.`); return parseCandidate(body, id); } +export async function updateCandidateRecord(product: string, candidate: Candidate, opts: LocalStoreOptions = {}): Promise { + const key = canonicalProductKey(product).key; + await writeProductFile(key, candidatePath(candidate.id), `${JSON.stringify(encodeCandidate(candidate), null, 2)}\n`, opts); +} + function candidatePath(id: string): string { if (!id || id.includes('/') || id.includes('\\') || id === '.' || id === '..' || id.startsWith('.')) { throw new Error(`Invalid site memory path: ${id}`); diff --git a/src/site-memory/checkpoint.test.ts b/src/site-memory/checkpoint.test.ts new file mode 100644 index 00000000..ec9b6ab1 --- /dev/null +++ b/src/site-memory/checkpoint.test.ts @@ -0,0 +1,446 @@ +import { execFile } from 'node:child_process'; +import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { promisify } from 'node:util'; +import { afterEach, describe, expect, it } from 'vitest'; +import { addCandidate, showCandidate } from './candidates.js'; +import { checkpointMemory, type CheckpointInput } from './checkpoint.js'; +import { openSitesRepository } from './git-store.js'; +import { readProductFile, writeProductFile } from './local-store.js'; + +const run = promisify(execFile); +const tempHomes: string[] = []; +const FACT = '- [verified 2026-08-31] Prefer /new for fresh posts.\n'; +const SITE = `# Example\n\n${FACT}`; +const POINTER = '- More: [references/listing.md](references/listing.md).\n'; +const REF = `# Listing\n\n${FACT}`; + +afterEach(async () => { + await Promise.all(tempHomes.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +describe('checkpoint compare-and-swap', () => { + it('returns a stale-revision conflict without copying the draft', async () => { + const { homeDir, revision } = await primed(); + await writeDraft(homeDir, { 'sitemap/SITE.md': `# Stale\n\n${FACT}` }); + + const result = await checkpoint(homeDir, { expectedRevision: '0'.repeat(40) }); + + expect(result).toEqual({ + status: 'conflict', + expectedRevision: '0'.repeat(40), + actualRevision: revision, + }); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBe(SITE); + }); + + it('accepts only contained Markdown sitemap paths', async () => { + const { homeDir } = await primed(); + await writeDraft(homeDir, { 'sitemap/SITE.md': SITE }); + + await expect(checkpoint(homeDir, { paths: ['manifest.json'] })).rejects.toThrow(/markdown|path/i); + await expect(checkpoint(homeDir, { paths: ['candidates/x.json'] })).rejects.toThrow(/markdown|path/i); + await expect(checkpoint(homeDir, { paths: ['sitemap/../manifest.json'] })).rejects.toThrow(/markdown|path|invalid/i); + await expect(checkpoint(homeDir, { paths: ['notes.md'] })).rejects.toThrow(/markdown|path/i); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBe(SITE); + }); + + it('requires a valid [verified YYYY-MM-DD] marker on every fact', async () => { + const { homeDir } = await primed(); + await writeDraft(homeDir, { 'sitemap/SITE.md': '# Example\n\n- Prefer /new for fresh posts.\n' }); + await expect(checkpoint(homeDir)).rejects.toThrow(/verified/i); + + await writeDraft(homeDir, { 'sitemap/SITE.md': '# Example\n\n- [verified 2026-13-40] Prefer /new.\n' }); + await expect(checkpoint(homeDir)).rejects.toThrow(/verified/i); + }); + + it('refuses a legacy beta SITE.md without copying the draft', async () => { + const { homeDir, sites } = await tempSites(); + const product = join(sites, 'example.test', 'sitemap'); + await mkdir(product, { recursive: true }); + const legacy = '---\nsite: example\nkind: site\nid: example\nstatus: verified\nverified_at: 2026-01-01\nsource: beta\n---\n# Beta\n'; + await writeFile(join(product, 'SITE.md'), legacy); + await git(sites, ['init']); + await git(sites, ['add', 'example.test/sitemap/SITE.md']); + await git(sites, ['-c', 'user.name=webcmd', '-c', 'user.email=webcmd@local', 'commit', '-m', 'legacy']); + const revision = (await git(sites, ['rev-parse', 'HEAD'])).trim(); + await writeDraft(homeDir, { 'sitemap/SITE.md': SITE }); + + await expect(checkpoint(homeDir, { expectedRevision: revision })).rejects.toThrow(/incompatible beta schema/i); + expect(await readFile(join(product, 'SITE.md'), 'utf8')).toBe(legacy); + }); +}); + +describe('checkpoint candidate dispositions', () => { + it('ingests pending candidates and rejects illegal transitions', async () => { + const { homeDir } = await primed(); + const first = await addCandidate(candidate(homeDir, { observedAt: '2026-08-30T12:00:00Z' })); + const second = await addCandidate(candidate(homeDir, { observedAt: '2026-08-31T12:00:00Z', claim: 'Later path' })); + const next = `# Example\n\n${FACT}- [verified 2026-08-31] Later path is denser.\n`; + await writeDraft(homeDir, { 'sitemap/SITE.md': next }); + + const result = await checkpoint(homeDir, { + dispositions: [ + { id: first.id, status: 'ingested', evidenceRole: 'supporting' }, + { id: second.id, status: 'ingested', evidenceRole: 'supporting' }, + ], + }); + expect(result.status).toBe('committed'); + expect((await showCandidate('example.test', first.id, { homeDir })).status).toBe('ingested'); + + await writeDraft(homeDir, { 'sitemap/SITE.md': `${next}- [verified 2026-08-31] Extra.\n` }, 'task-2'); + await expect(checkpoint(homeDir, { + taskId: 'task-2', + dispositions: [{ id: first.id, status: 'rejected', rejectionReason: 'stale' }], + })).rejects.toThrow(/pending|transition|status/i); + }); + + it('requires two distinct UTC dates and rejects same-date evidence', async () => { + const { homeDir } = await primed(); + const a = await addCandidate(candidate(homeDir, { observedAt: '2026-08-31T01:00:00Z' })); + const b = await addCandidate(candidate(homeDir, { observedAt: '2026-08-31T23:00:00Z', claim: 'Same day' })); + await writeDraft(homeDir, { 'sitemap/SITE.md': `# Example\n\n${FACT}- [verified 2026-08-31] Same day is not enough.\n` }); + + await expect(checkpoint(homeDir, { + dispositions: [ + { id: a.id, status: 'ingested', evidenceRole: 'supporting' }, + { id: b.id, status: 'ingested', evidenceRole: 'supporting' }, + ], + })).rejects.toThrow(/date/i); + expect((await showCandidate('example.test', a.id, { homeDir })).status).toBe('pending'); + }); + + it('ingests a non-conflicting high-consequence candidate immediately', async () => { + const { homeDir } = await primed(); + const warning = await addCandidate(candidate(homeDir, { + kind: 'high_consequence', + claim: 'Ban risk on bulk delete', + observedAt: '2026-08-31T12:00:00Z', + })); + const next = `# Example\n\n${FACT}- [verified 2026-08-31] Bulk delete can ban the account.\n`; + await writeDraft(homeDir, { 'sitemap/SITE.md': next }); + + const result = await checkpoint(homeDir, { + dispositions: [{ id: warning.id, status: 'ingested', evidenceRole: 'supporting' }], + }); + + expect(result.status).toBe('committed'); + expect((await showCandidate('example.test', warning.id, { homeDir })).status).toBe('ingested'); + }); + + it('delays a conflicting high-consequence candidate until a later UTC date', async () => { + const { homeDir } = await primed(); + const first = await addCandidate(candidate(homeDir, { + kind: 'high_consequence', + claim: 'Ban risk on bulk delete', + observedAt: '2026-08-30T12:00:00Z', + })); + await writeDraft(homeDir, { 'sitemap/SITE.md': `# Example\n\n${FACT}- [verified 2026-08-31] Overturn the safe-delete claim.\n` }); + + await expect(checkpoint(homeDir, { + dispositions: [{ id: first.id, status: 'ingested', evidenceRole: 'supporting', conflictsWithMemory: true }], + })).rejects.toThrow(/date|conflict/i); + expect((await showCandidate('example.test', first.id, { homeDir })).status).toBe('pending'); + + const later = await addCandidate(candidate(homeDir, { + kind: 'high_consequence', + claim: 'Ban risk confirmed later', + observedAt: '2026-08-31T12:00:00Z', + })); + const result = await checkpoint(homeDir, { + dispositions: [ + { id: first.id, status: 'ingested', evidenceRole: 'supporting', conflictsWithMemory: true }, + { id: later.id, status: 'ingested', evidenceRole: 'supporting' }, + ], + }); + expect(result.status).toBe('committed'); + }); + + it('checkpoints a direct correction without candidate promotion', async () => { + const { homeDir } = await primed(); + const pending = await addCandidate(candidate(homeDir)); + const next = `# Example\n\n- [verified 2026-09-01] /hot is the live listing.\n`; + await writeDraft(homeDir, { 'sitemap/SITE.md': next }); + + const result = await checkpoint(homeDir, { reason: 'direct_correction', dispositions: [] }); + + expect(result.status).toBe('committed'); + if (result.status === 'committed') expect(result.provenanceCommit).toBeNull(); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBe(next); + expect((await showCandidate('example.test', pending.id, { homeDir })).status).toBe('pending'); + }); + + it('records supporting and dissenting roles and requires rejection reasons', async () => { + const { homeDir } = await primed(); + const support = await addCandidate(candidate(homeDir, { observedAt: '2026-08-30T12:00:00Z' })); + const dissent = await addCandidate(candidate(homeDir, { observedAt: '2026-08-31T12:00:00Z', claim: 'Disagree' })); + const junk = await addCandidate(candidate(homeDir, { observedAt: '2026-08-29T12:00:00Z', claim: 'Transient' })); + await writeDraft(homeDir, { 'sitemap/SITE.md': `# Example\n\n${FACT}- [verified 2026-08-31] Keep /new.\n` }); + + await expect(checkpoint(homeDir, { + dispositions: [{ id: junk.id, status: 'rejected' }], + })).rejects.toThrow(/reason/i); + + const result = await checkpoint(homeDir, { + dispositions: [ + { id: support.id, status: 'ingested', evidenceRole: 'supporting' }, + { id: dissent.id, status: 'ingested', evidenceRole: 'dissenting' }, + { id: junk.id, status: 'rejected', rejectionReason: 'transient' }, + ], + }); + expect(result.status).toBe('committed'); + const storedSupport = await showCandidate('example.test', support.id, { homeDir }); + const storedDissent = await showCandidate('example.test', dissent.id, { homeDir }); + const storedJunk = await showCandidate('example.test', junk.id, { homeDir }); + expect(storedSupport.evidenceRole).toBe('supporting'); + expect(storedDissent.evidenceRole).toBe('dissenting'); + expect(storedJunk).toMatchObject({ status: 'rejected', evidenceRole: null, rejectionReason: 'transient' }); + const raw = JSON.parse(await readProductFile('example.test', `candidates/${support.id}.json`, { homeDir }) ?? ''); + expect(raw.memory_commit).toMatch(/^[0-9a-f]{40}$/); + expect(raw.evidence_role).toBe('supporting'); + expect(raw.schema_version).toBe(1); + }); +}); + +describe('checkpoint git transaction', () => { + it('commits memory before provenance, stages explicit paths, and leaves candidates pending if memory commit fails', async () => { + const { homeDir, sites } = await primed(); + const first = await addCandidate(candidate(homeDir, { observedAt: '2026-08-30T12:00:00Z' })); + const second = await addCandidate(candidate(homeDir, { observedAt: '2026-08-31T12:00:00Z', claim: 'Later' })); + await writeFile(join(sites, 'example.test', 'scratch.md'), 'unrelated\n'); + const next = `# Example\n\n${FACT}- [verified 2026-08-31] Later path is denser.\n`; + await writeDraft(homeDir, { 'sitemap/SITE.md': next }); + + const result = await checkpoint(homeDir, { + dispositions: [ + { id: first.id, status: 'ingested', evidenceRole: 'supporting' }, + { id: second.id, status: 'ingested', evidenceRole: 'supporting' }, + ], + }); + expect(result.status).toBe('committed'); + if (result.status !== 'committed') throw new Error('expected commit'); + + const memoryFiles = (await git(sites, ['show', '--name-only', '--pretty=format:', result.memoryCommit])).trim().split('\n'); + const provenanceFiles = (await git(sites, ['show', '--name-only', '--pretty=format:', result.provenanceCommit ?? ''])).trim().split('\n'); + expect(memoryFiles).toContain('example.test/sitemap/SITE.md'); + expect(memoryFiles.join('\n')).not.toMatch(/candidates/); + expect(provenanceFiles.join('\n')).toMatch(/candidates/); + expect(provenanceFiles.join('\n')).not.toMatch(/SITE\.md/); + expect((await git(sites, ['log', '--format=%H', `${result.memoryCommit}..${result.provenanceCommit}`])).trim()).toBe(result.provenanceCommit); + expect((await git(sites, ['ls-files'])).trim().split('\n')).not.toContain('example.test/scratch.md'); + + const blocked = await primed(); + const pending = await addCandidate(candidate(blocked.homeDir, { observedAt: '2026-08-30T12:00:00Z' })); + const later = await addCandidate(candidate(blocked.homeDir, { observedAt: '2026-08-31T12:00:00Z', claim: 'Later' })); + await writeDraft(blocked.homeDir, { 'sitemap/SITE.md': next }); + await withGitWrapper(blocked.homeDir, 'memory', async () => { + await expect(checkpoint(blocked.homeDir, { + dispositions: [ + { id: pending.id, status: 'ingested', evidenceRole: 'supporting' }, + { id: later.id, status: 'ingested', evidenceRole: 'supporting' }, + ], + })).rejects.toThrow(); + }); + expect((await showCandidate('example.test', pending.id, { homeDir: blocked.homeDir })).status).toBe('pending'); + expect((await showCandidate('example.test', pending.id, { homeDir: blocked.homeDir })).memoryCommit).toBeNull(); + expect((await showCandidate('example.test', later.id, { homeDir: blocked.homeDir })).status).toBe('pending'); + }); + + it('resumes a failed provenance commit without replaying the memory change', async () => { + const { homeDir, sites } = await primed(); + const first = await addCandidate(candidate(homeDir, { observedAt: '2026-08-30T12:00:00Z' })); + const second = await addCandidate(candidate(homeDir, { observedAt: '2026-08-31T12:00:00Z', claim: 'Later' })); + const next = `# Example\n\n${FACT}- [verified 2026-08-31] Later path is denser.\n`; + await writeDraft(homeDir, { 'sitemap/SITE.md': next }); + const dispositions = [ + { id: first.id, status: 'ingested' as const, evidenceRole: 'supporting' as const }, + { id: second.id, status: 'ingested' as const, evidenceRole: 'supporting' as const }, + ]; + + await withGitWrapper(homeDir, 'provenance', async () => { + await expect(checkpoint(homeDir, { dispositions })).rejects.toThrow(); + }); + + const memoryRevision = (await git(sites, ['rev-parse', 'HEAD'])).trim(); + expect(await git(sites, ['show', `HEAD:example.test/sitemap/SITE.md`])).toBe(next); + expect((await git(sites, ['log', '--oneline', '--', 'example.test/sitemap/SITE.md'])).trim().split('\n')).toHaveLength(2); + const disk = await showCandidate('example.test', first.id, { homeDir }); + expect(disk.status).toBe('ingested'); + expect(disk.memoryCommit).toBe(memoryRevision); + + const resumed = await checkpoint(homeDir, { expectedRevision: memoryRevision, dispositions }); + expect(resumed.status).toBe('committed'); + if (resumed.status !== 'committed') throw new Error('expected commit'); + expect(resumed.memoryCommit).toBe(memoryRevision); + expect(resumed.provenanceCommit).toMatch(/^[0-9a-f]{40}$/); + expect(resumed.provenanceCommit).not.toBe(memoryRevision); + expect((await git(sites, ['log', '--oneline', '--', 'example.test/sitemap/SITE.md'])).trim().split('\n')).toHaveLength(2); + expect((await git(sites, ['show', `HEAD:example.test/candidates/${first.id}.json`]))).toMatch(/"status": "ingested"/); + }); +}); + +describe('checkpoint rewrite bounds', () => { + it('allows an unchanged oversized seed', async () => { + const oversized = siteLines(501); + const { homeDir } = await primed(oversized); + await writeDraft(homeDir, { 'sitemap/SITE.md': oversized }); + + const result = await checkpoint(homeDir, { reason: 'direct_correction', dispositions: [] }); + + expect(result.status).toBe('committed'); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBe(oversized); + }); + + it('requires a later update of an oversized SITE.md to be at most 200 lines', async () => { + const { homeDir } = await primed(siteLines(501)); + await writeDraft(homeDir, { 'sitemap/SITE.md': siteLines(500) }); + await expect(checkpoint(homeDir, { reason: 'direct_correction', dispositions: [] })).rejects.toThrow(/200|rewrite/i); + + const rewritten = siteLines(200, true); + await writeDraft(homeDir, { + 'sitemap/SITE.md': rewritten, + 'sitemap/references/listing.md': REF, + }); + const result = await checkpoint(homeDir, { + reason: 'major_rewrite', + paths: ['sitemap/SITE.md', 'sitemap/references/listing.md'], + dispositions: [], + }); + expect(result.status).toBe('committed'); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBe(rewritten); + }); + + it('preserves the 200-line cap and contextual pointers after a rewrite', async () => { + const { homeDir, revision } = await primed(siteLines(501)); + await writeDraft(homeDir, { + 'sitemap/SITE.md': siteLines(200, true), + 'sitemap/references/listing.md': REF, + }); + const rewritten = await checkpoint(homeDir, { + expectedRevision: revision, + reason: 'major_rewrite', + paths: ['sitemap/SITE.md', 'sitemap/references/listing.md'], + dispositions: [], + }); + expect(rewritten.status).toBe('committed'); + if (rewritten.status !== 'committed') throw new Error('expected commit'); + + await writeDraft(homeDir, { 'sitemap/SITE.md': siteLines(201, true) }, 'task-2'); + await expect(checkpoint(homeDir, { + taskId: 'task-2', + expectedRevision: rewritten.memoryCommit, + reason: 'direct_correction', + dispositions: [], + })).rejects.toThrow(/200/i); + + const next = siteLines(180, true); + await writeDraft(homeDir, { 'sitemap/SITE.md': next }, 'task-2'); + const result = await checkpoint(homeDir, { + taskId: 'task-2', + expectedRevision: rewritten.memoryCommit, + reason: 'direct_correction', + dispositions: [], + }); + expect(result.status).toBe('committed'); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBe(next); + expect(next).toContain('](references/listing.md)'); + }); +}); + +function siteLines(count: number, pointer = false): string { + const extra = pointer ? [POINTER.trimEnd()] : []; + const heading = ['# Example']; + const facts = Array.from({ length: count - heading.length - extra.length }, (_, i) => `- [verified 2026-08-31] Fact ${i}.`); + return `${[...heading, ...facts, ...extra].join('\n')}\n`; +} + +async function checkpoint(homeDir: string, extra: Partial = {}) { + return checkpointMemory({ + product: 'example.test', + taskId: 'task-1', + reason: 'candidate_ingestion', + paths: ['sitemap/SITE.md'], + dispositions: [], + ...extra, + homeDir, + expectedRevision: extra.expectedRevision ?? await (await openSitesRepository({ homeDir })).revision(), + }); +} + +function candidate(homeDir: string, extra: Record = {}) { + return { + product: 'example.test', + hostname: 'www.example.test', + kind: 'better_path', + claim: 'New listing is faster', + evidence: 'Used /new while /hot spun.', + consequence: 'Prefer /new for fresh posts', + environment: {}, + homeDir, + ...extra, + }; +} + +async function primed(site = SITE) { + const { homeDir, sites } = await tempSites(); + const product = { + key: 'example.test', + hostname: 'example.test', + displayHostname: 'example.test', + registrableDomain: 'example.test', + }; + await writeProductFile('example.test', 'manifest.json', `${JSON.stringify({ + schemaVersion: 1, + product, + interfaces: [], + seed: { status: 'absent' }, + }, null, 2)}\n`, { homeDir }); + await writeProductFile('example.test', 'sitemap/SITE.md', site, { homeDir }); + const repo = await openSitesRepository({ homeDir }); + const revision = await repo.commit(['example.test/manifest.json', 'example.test/sitemap/SITE.md'], 'init'); + return { homeDir, sites, revision, repo }; +} + +async function writeDraft(homeDir: string, files: Record, taskId = 'task-1') { + for (const [path, body] of Object.entries(files)) { + const target = join(homeDir, '.webcmd/sites/.drafts', taskId, 'example.test', path); + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, body); + } +} + +async function withGitWrapper(homeDir: string, fail: 'memory' | 'provenance', fn: () => Promise) { + const originalPath = process.env.PATH; + const wrapperDir = await mkdtemp(join(tmpdir(), 'webcmd-checkpoint-git-')); + tempHomes.push(wrapperDir); + const { stdout } = await run('/usr/bin/which', ['git'], { encoding: 'utf8' }); + const needle = fail === 'memory' ? 'checkpoint memory' : 'checkpoint provenance'; + const wrapper = join(wrapperDir, 'git'); + await writeFile(wrapper, `#!/usr/bin/env node +const { spawnSync } = require('node:child_process'); +const args = process.argv.slice(2); +const msg = args.includes('-m') ? args[args.indexOf('-m') + 1] : ''; +if (args.includes('commit') && msg.includes(${JSON.stringify(needle)})) process.exit(1); +const result = spawnSync(${JSON.stringify(stdout.trim())}, args, { stdio: 'inherit' }); +process.exit(result.status ?? 1); +`); + await chmod(wrapper, 0o755); + process.env.PATH = `${wrapperDir}:${originalPath}`; + try { + await fn(); + } finally { + process.env.PATH = originalPath; + } +} + +async function tempSites() { + const homeDir = await mkdtemp(join(tmpdir(), 'webcmd-checkpoint-')); + tempHomes.push(homeDir); + return { homeDir, sites: join(homeDir, '.webcmd', 'sites') }; +} + +async function git(cwd: string, args: string[]) { + const { stdout } = await run('git', args, { cwd, encoding: 'utf8' }); + return stdout; +} diff --git a/src/site-memory/checkpoint.ts b/src/site-memory/checkpoint.ts new file mode 100644 index 00000000..552c3f09 --- /dev/null +++ b/src/site-memory/checkpoint.ts @@ -0,0 +1,246 @@ +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { readCandidateRecord, updateCandidateRecord } from './candidates.js'; +import { openSitesRepository } from './git-store.js'; +import { containedRelativePath, copyDraftFiles, readProductFile, sitesRoot, type LocalStoreOptions } from './local-store.js'; +import type { + Candidate, + CandidateDisposition, + CheckpointReason, + CheckpointResult, + MemoryRevision, +} from './model.js'; +import { canonicalProductKey } from './product-resolver.js'; + +export interface CheckpointInput extends LocalStoreOptions { + product: string; + taskId: string; + expectedRevision: MemoryRevision | null; + reason: CheckpointReason; + paths: string[]; + dispositions?: CandidateDisposition[]; +} + +const REASONS = new Set(['candidate_ingestion', 'direct_correction', 'major_rewrite']); +const VERIFIED = /\[verified (\d{4}-\d{2}-\d{2})\]/; +const POINTER = /\]\(references\/[^)]+\)/; + +export async function checkpointMemory(input: CheckpointInput): Promise { + if (!REASONS.has(input.reason)) throw new Error(`Invalid checkpoint reason: ${input.reason}`); + const paths = input.paths.map(assertMarkdownPath); + const product = canonicalProductKey(input.product).key; + const taskId = memorySegment(input.taskId); + const dispositions = input.dispositions ?? []; + const repo = await openSitesRepository(input); + + return repo.withRepositoryLock(async () => { + const actual = await repo.revision(); + const loaded = await Promise.all(dispositions.map((row) => readCandidateRecord(product, row.id, input))); + + if (actual && input.expectedRevision === actual && isIncompleteProvenance(loaded, dispositions, actual)) { + await writeDispositions(product, loaded, dispositions, actual, input); + const provenanceCommit = await repo.commit( + dispositions.map((row) => `${product}/candidates/${row.id}.json`), + `checkpoint provenance ${product}`, + ); + return { status: 'committed', memoryCommit: actual, provenanceCommit }; + } + + if (actual !== input.expectedRevision) { + return { status: 'conflict', expectedRevision: input.expectedRevision, actualRevision: actual }; + } + + if (await isLegacy(product, input)) { + throw new Error('Incompatible beta schema; learning is read-only until this SITE.md is cleared.'); + } + + const drafts = await readDrafts(input, taskId, product, paths); + for (const body of drafts.values()) validateFacts(body); + validateLineBounds(await readProductFile(product, 'sitemap/SITE.md', input), drafts.get('sitemap/SITE.md'), input.reason); + validateDispositions(loaded, dispositions); + + const changed: string[] = []; + for (const path of paths) { + if (drafts.get(path) !== await readProductFile(product, path, input)) changed.push(path); + } + let memoryCommit = actual; + if (changed.length > 0) { + await copyDraftFiles(product, taskId, paths, input); + memoryCommit = await repo.commit(paths.map((path) => `${product}/${path}`), `checkpoint memory ${product}`); + } + if (!memoryCommit) throw new Error('Refusing to checkpoint without a memory revision.'); + + if (dispositions.length === 0) return { status: 'committed', memoryCommit, provenanceCommit: null }; + + await writeDispositions(product, loaded, dispositions, memoryCommit, input); + const provenanceCommit = await repo.commit( + dispositions.map((row) => `${product}/candidates/${row.id}.json`), + `checkpoint provenance ${product}`, + ); + return { status: 'committed', memoryCommit, provenanceCommit }; + }); +} + +function isIncompleteProvenance( + loaded: Candidate[], + dispositions: CandidateDisposition[], + actual: MemoryRevision, +): boolean { + if (dispositions.length === 0) return false; + return dispositions.every((row, index) => { + const candidate = loaded[index]; + if (candidate.status !== row.status) return false; + if (row.status === 'ingested') return candidate.memoryCommit === actual; + return candidate.reviewedAt !== null && candidate.rejectionReason !== null; + }); +} + +async function writeDispositions( + product: string, + loaded: Candidate[], + dispositions: CandidateDisposition[], + memoryCommit: MemoryRevision, + opts: LocalStoreOptions, +): Promise { + const reviewedAt = new Date().toISOString(); + for (const [index, row] of dispositions.entries()) { + const current = loaded[index]; + const next: Candidate = row.status === 'ingested' + ? { + ...current, + status: 'ingested', + evidenceRole: row.evidenceRole === 'supporting' || row.evidenceRole === 'dissenting' ? row.evidenceRole : current.evidenceRole, + memoryCommit, + reviewedAt: current.reviewedAt ?? reviewedAt, + rejectionReason: null, + } + : { + ...current, + status: 'rejected', + evidenceRole: null, + memoryCommit: null, + reviewedAt: current.reviewedAt ?? reviewedAt, + rejectionReason: row.rejectionReason ?? current.rejectionReason, + }; + await updateCandidateRecord(product, next, opts); + } +} + +function validateDispositions(loaded: Candidate[], dispositions: CandidateDisposition[]): void { + for (const [index, row] of dispositions.entries()) { + if (loaded[index].status !== 'pending') throw new Error('Invalid candidate status transition.'); + if (row.status === 'ingested') { + if (row.evidenceRole !== 'supporting' && row.evidenceRole !== 'dissenting') { + throw new Error('Invalid candidate evidence_role.'); + } + if (row.rejectionReason) throw new Error('Invalid candidate status.'); + } else if (row.status === 'rejected') { + if (!row.rejectionReason?.trim()) throw new Error('Rejected candidates require a reason.'); + if (row.evidenceRole) throw new Error('Invalid candidate status.'); + } else { + throw new Error('Invalid candidate status.'); + } + } + const ingested = dispositions.map((row, index) => ({ row, candidate: loaded[index] })).filter((entry) => entry.row.status === 'ingested'); + if (ingested.length === 0) return; + const dates = [...new Set(ingested.map((entry) => entry.candidate.observedDateUtc))]; + const conflicting = ingested.filter((entry) => entry.row.conflictsWithMemory); + if (conflicting.length > 0) { + const first = conflicting.map((entry) => entry.candidate.observedDateUtc).sort()[0]; + if (!dates.some((date) => date > first)) { + throw new Error('Conflicting high-consequence evidence requires a later UTC date.'); + } + return; + } + if (ingested.every((entry) => entry.candidate.kind === 'high_consequence')) return; + if (dates.length < 2) throw new Error('Ingestion requires evidence on two distinct UTC dates.'); +} + +function validateLineBounds(current: string | null, draft: string | undefined, reason: CheckpointReason): void { + if (draft === undefined || (current ?? '') === draft) return; + const currentLines = physicalLines(current ?? ''); + const draftLines = physicalLines(draft); + const draftPointers = POINTER.test(draft); + if (currentLines > 500 || reason === 'major_rewrite') { + if (draftLines > 200) throw new Error('SITE.md updates over 500 lines require a rewrite to at most 200 lines.'); + if (!draftPointers) throw new Error('A major rewrite requires contextual reference pointers.'); + return; + } + // ponytail: pointer-bearing SITE.md is treated as post-rewrite and capped at 200; persist a rewrite flag if organic 201-500 growth with pointers is required + if (currentLines <= 200 && POINTER.test(current ?? '')) { + if (draftLines > 200) throw new Error('Post-rewrite SITE.md updates must stay at or below 200 lines.'); + if (!draftPointers) throw new Error('Post-rewrite updates require contextual reference pointers.'); + return; + } + if (draftLines > 500) throw new Error('SITE.md updates over 500 lines require a rewrite to at most 200 lines.'); +} + +function validateFacts(body: string): void { + for (const line of body.split('\n')) { + const text = line.trim(); + if (!text || /^#{1,6}\s/.test(text) || (POINTER.test(text) && !VERIFIED.test(text))) continue; + const match = VERIFIED.exec(text); + if (!match || !validUtcDate(match[1])) { + throw new Error('Each durable fact requires a valid [verified YYYY-MM-DD] date.'); + } + } +} + +function validUtcDate(text: string): boolean { + const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(text); + if (!match) return false; + const year = Number(match[1]); + const month = Number(match[2]); + const day = Number(match[3]); + const date = new Date(Date.UTC(year, month - 1, day)); + return date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day; +} + +function physicalLines(text: string): number { + if (text === '') return 0; + return text.endsWith('\n') ? text.slice(0, -1).split('\n').length : text.split('\n').length; +} + +function assertMarkdownPath(path: string): string { + const normalized = path.split('\\').join('/'); + if (normalized.split('/').some((part) => part === '.' || part === '..' || part === '')) { + throw new Error(`Invalid site memory path: ${path}`); + } + if (normalized === 'sitemap/SITE.md') return normalized; + if (/^sitemap\/references\/[^./][^/]*\.md$/.test(normalized)) return normalized; + throw new Error(`Invalid site memory Markdown path: ${path}`); +} + +async function isLegacy(product: string, opts: LocalStoreOptions): Promise { + if (!await readProductFile(product, 'sitemap/SITE.md', opts)) return false; + const raw = await readProductFile(product, 'manifest.json', opts); + if (!raw) return true; + try { + const value = JSON.parse(raw) as { schemaVersion?: unknown }; + return value.schemaVersion !== 1; + } catch { + return true; + } +} + +async function readDrafts( + input: LocalStoreOptions, + taskId: string, + product: string, + paths: string[], +): Promise> { + const root = join(sitesRoot(input), '.drafts', taskId, product); + const drafts = new Map(); + for (const path of paths) { + const relative = containedRelativePath(root, path); + drafts.set(path, await readFile(join(root, ...relative.split('/')), 'utf8')); + } + return drafts; +} + +function memorySegment(value: string): string { + if (!value || value.includes('/') || value.includes('\\') || value === '.' || value === '..' || value.startsWith('.')) { + throw new Error(`Invalid site memory path: ${value}`); + } + return value; +} diff --git a/src/site-memory/model.ts b/src/site-memory/model.ts index cef658fa..a70dfb3e 100644 --- a/src/site-memory/model.ts +++ b/src/site-memory/model.ts @@ -100,3 +100,17 @@ export interface CandidateSummary { consequence: string; status: CandidateStatus; } + +export type CheckpointReason = 'candidate_ingestion' | 'direct_correction' | 'major_rewrite'; + +export interface CandidateDisposition { + id: string; + status: 'ingested' | 'rejected'; + evidenceRole?: 'supporting' | 'dissenting' | null; + rejectionReason?: string | null; + conflictsWithMemory?: boolean; +} + +export type CheckpointResult = + | { status: 'committed'; memoryCommit: MemoryRevision; provenanceCommit: MemoryRevision | null } + | { status: 'conflict'; expectedRevision: MemoryRevision | null; actualRevision: MemoryRevision | null }; From aee4cdc683879ae99c74012ef3adf518e5955a90 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Tue, 1 Sep 2026 00:53:49 +0530 Subject: [PATCH 13/41] fix(site-memory): roll back failed memory commits and tighten checkpoint Restore prior Markdown and delete new paths if the memory commit fails, combining rollback errors. Reuse parseProductManifest, recover partial provenance without replaying memory, and require ingestion dispositions plus a real memory change. --- src/site-memory/checkpoint.test.ts | 212 +++++++++++++++++++++++++++- src/site-memory/checkpoint.ts | 111 ++++++++++++--- src/site-memory/context.test.ts | 20 ++- src/site-memory/context.ts | 8 +- src/site-memory/local-store.test.ts | 13 ++ src/site-memory/local-store.ts | 11 ++ 6 files changed, 345 insertions(+), 30 deletions(-) diff --git a/src/site-memory/checkpoint.test.ts b/src/site-memory/checkpoint.test.ts index ec9b6ab1..a1d33ba8 100644 --- a/src/site-memory/checkpoint.test.ts +++ b/src/site-memory/checkpoint.test.ts @@ -70,6 +70,35 @@ describe('checkpoint compare-and-swap', () => { await expect(checkpoint(homeDir, { expectedRevision: revision })).rejects.toThrow(/incompatible beta schema/i); expect(await readFile(join(product, 'SITE.md'), 'utf8')).toBe(legacy); }); + + it.each([ + ['missing', null], + ['malformed', '{'], + ['non-v1', JSON.stringify({ + schemaVersion: 2, + product: { key: 'example.test', hostname: 'example.test', displayHostname: 'example.test', registrableDomain: 'example.test' }, + interfaces: [], + seed: { status: 'absent' }, + })], + ['partial', JSON.stringify({ + schemaVersion: 1, + product: { key: 'example.test', hostname: 'example.test', displayHostname: 'example.test', registrableDomain: 'example.test' }, + seed: { status: 'absent' }, + })], + ] as const)('refuses a %s product manifest without copying the draft', async (_label, body) => { + const { homeDir, sites, revision: initial } = await primed(); + const manifest = join(sites, 'example.test', 'manifest.json'); + if (body === null) await rm(manifest); + else await writeFile(manifest, `${body}\n`); + await git(sites, ['add', '-A', 'example.test/manifest.json']); + await git(sites, ['-c', 'user.name=webcmd', '-c', 'user.email=webcmd@local', 'commit', '-m', 'bad-manifest']); + const revision = (await git(sites, ['rev-parse', 'HEAD'])).trim(); + expect(revision).not.toBe(initial); + await writeDraft(homeDir, { 'sitemap/SITE.md': `# Next\n\n${FACT}` }); + + await expect(checkpoint(homeDir, { expectedRevision: revision, reason: 'direct_correction' })).rejects.toThrow(/incompatible beta schema|manifest|schema/i); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBe(SITE); + }); }); describe('checkpoint candidate dispositions', () => { @@ -171,6 +200,66 @@ describe('checkpoint candidate dispositions', () => { expect((await showCandidate('example.test', pending.id, { homeDir })).status).toBe('pending'); }); + it('requires candidate_ingestion dispositions and a memory change', async () => { + const { homeDir } = await primed(); + const first = await addCandidate(candidate(homeDir, { observedAt: '2026-08-30T12:00:00Z' })); + const second = await addCandidate(candidate(homeDir, { observedAt: '2026-08-31T12:00:00Z', claim: 'Later' })); + await writeDraft(homeDir, { 'sitemap/SITE.md': `# Example\n\n${FACT}- [verified 2026-08-31] Later path is denser.\n` }); + + await expect(checkpoint(homeDir)).rejects.toThrow(/disposition/i); + await writeDraft(homeDir, { 'sitemap/SITE.md': SITE }); + await expect(checkpoint(homeDir, { + dispositions: [ + { id: first.id, status: 'ingested', evidenceRole: 'supporting' }, + { id: second.id, status: 'ingested', evidenceRole: 'supporting' }, + ], + })).rejects.toThrow(/memory|change|unchanged/i); + expect((await showCandidate('example.test', first.id, { homeDir })).status).toBe('pending'); + }); + + it('rejects dispositions on direct_correction and major_rewrite', async () => { + const { homeDir } = await primed(); + const pending = await addCandidate(candidate(homeDir)); + await writeDraft(homeDir, { 'sitemap/SITE.md': `# Example\n\n- [verified 2026-09-01] /hot is the live listing.\n` }); + + await expect(checkpoint(homeDir, { + reason: 'direct_correction', + dispositions: [{ id: pending.id, status: 'rejected', rejectionReason: 'stale' }], + })).rejects.toThrow(/disposition/i); + await writeDraft(homeDir, { + 'sitemap/SITE.md': siteLines(200, true), + 'sitemap/references/listing.md': REF, + }); + await expect(checkpoint(homeDir, { + reason: 'major_rewrite', + paths: ['sitemap/SITE.md', 'sitemap/references/listing.md'], + dispositions: [{ id: pending.id, status: 'rejected', rejectionReason: 'stale' }], + })).rejects.toThrow(/disposition/i); + expect((await showCandidate('example.test', pending.id, { homeDir })).status).toBe('pending'); + }); + + it('rejects duplicate Markdown paths and duplicate candidate ids', async () => { + const { homeDir } = await primed(); + const first = await addCandidate(candidate(homeDir, { observedAt: '2026-08-30T12:00:00Z' })); + const second = await addCandidate(candidate(homeDir, { observedAt: '2026-08-31T12:00:00Z', claim: 'Later' })); + await writeDraft(homeDir, { 'sitemap/SITE.md': `# Example\n\n${FACT}- [verified 2026-08-31] Later path is denser.\n` }); + + await expect(checkpoint(homeDir, { + paths: ['sitemap/SITE.md', 'sitemap/SITE.md'], + dispositions: [ + { id: first.id, status: 'ingested', evidenceRole: 'supporting' }, + { id: second.id, status: 'ingested', evidenceRole: 'supporting' }, + ], + })).rejects.toThrow(/duplicate|path/i); + await expect(checkpoint(homeDir, { + dispositions: [ + { id: first.id, status: 'ingested', evidenceRole: 'supporting' }, + { id: first.id, status: 'ingested', evidenceRole: 'supporting' }, + ], + })).rejects.toThrow(/duplicate|id/i); + expect((await showCandidate('example.test', first.id, { homeDir })).status).toBe('pending'); + }); + it('records supporting and dissenting roles and requires rejection reasons', async () => { const { homeDir } = await primed(); const support = await addCandidate(candidate(homeDir, { observedAt: '2026-08-30T12:00:00Z' })); @@ -247,6 +336,56 @@ describe('checkpoint git transaction', () => { expect((await showCandidate('example.test', later.id, { homeDir: blocked.homeDir })).status).toBe('pending'); }); + it('restores prior Markdown, deletes new paths, and keeps the index clean if memory commit fails', async () => { + const blocked = await primed(); + const pending = await addCandidate(candidate(blocked.homeDir, { observedAt: '2026-08-30T12:00:00Z' })); + const later = await addCandidate(candidate(blocked.homeDir, { observedAt: '2026-08-31T12:00:00Z', claim: 'Later' })); + const next = `# Example\n\n${FACT}- [verified 2026-08-31] Later path is denser.\n`; + await writeDraft(blocked.homeDir, { + 'sitemap/SITE.md': next, + 'sitemap/references/listing.md': REF, + }); + + await withGitWrapper(blocked.homeDir, 'memory', async () => { + await expect(checkpoint(blocked.homeDir, { + paths: ['sitemap/SITE.md', 'sitemap/references/listing.md'], + dispositions: [ + { id: pending.id, status: 'ingested', evidenceRole: 'supporting' }, + { id: later.id, status: 'ingested', evidenceRole: 'supporting' }, + ], + })).rejects.toThrow(); + }); + + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir: blocked.homeDir })).toBe(SITE); + expect(await readProductFile('example.test', 'sitemap/references/listing.md', { homeDir: blocked.homeDir })).toBeNull(); + expect((await showCandidate('example.test', pending.id, { homeDir: blocked.homeDir })).status).toBe('pending'); + expect((await showCandidate('example.test', later.id, { homeDir: blocked.homeDir })).status).toBe('pending'); + expect((await git(blocked.sites, ['status', '--porcelain', '-uall'])).trim()).toBe(''); + expect((await git(blocked.sites, ['diff', '--cached', '--name-only'])).trim()).toBe(''); + }); + + it('combines memory commit and rollback errors', async () => { + const blocked = await primed(); + const pending = await addCandidate(candidate(blocked.homeDir, { observedAt: '2026-08-30T12:00:00Z' })); + const later = await addCandidate(candidate(blocked.homeDir, { observedAt: '2026-08-31T12:00:00Z', claim: 'Later' })); + const next = `# Example\n\n${FACT}- [verified 2026-08-31] Later path is denser.\n`; + await writeDraft(blocked.homeDir, { 'sitemap/SITE.md': next }); + const sitemapDir = join(blocked.sites, 'example.test', 'sitemap'); + + try { + await withGitWrapper(blocked.homeDir, 'memory', async () => { + await expect(checkpoint(blocked.homeDir, { + dispositions: [ + { id: pending.id, status: 'ingested', evidenceRole: 'supporting' }, + { id: later.id, status: 'ingested', evidenceRole: 'supporting' }, + ], + })).rejects.toThrow(/checkpoint memory[\s\S]*(EACCES|EPERM|permission denied)|(EACCES|EPERM|permission denied)[\s\S]*checkpoint memory/i); + }, sitemapDir); + } finally { + await chmod(sitemapDir, 0o755).catch(() => undefined); + } + }); + it('resumes a failed provenance commit without replaying the memory change', async () => { const { homeDir, sites } = await primed(); const first = await addCandidate(candidate(homeDir, { observedAt: '2026-08-30T12:00:00Z' })); @@ -278,6 +417,71 @@ describe('checkpoint git transaction', () => { expect((await git(sites, ['log', '--oneline', '--', 'example.test/sitemap/SITE.md'])).trim().split('\n')).toHaveLength(2); expect((await git(sites, ['show', `HEAD:example.test/candidates/${first.id}.json`]))).toMatch(/"status": "ingested"/); }); + + it('finishes remaining provenance writes without replaying memory', async () => { + const { homeDir, sites } = await primed(); + const first = await addCandidate(candidate(homeDir, { observedAt: '2026-08-30T12:00:00Z' })); + const second = await addCandidate(candidate(homeDir, { observedAt: '2026-08-31T12:00:00Z', claim: 'Later' })); + const next = `# Example\n\n${FACT}- [verified 2026-08-31] Later path is denser.\n`; + await writeDraft(homeDir, { 'sitemap/SITE.md': next }); + const dispositions = [ + { id: first.id, status: 'ingested' as const, evidenceRole: 'supporting' as const }, + { id: second.id, status: 'ingested' as const, evidenceRole: 'supporting' as const }, + ]; + + await withGitWrapper(homeDir, 'provenance', async () => { + await expect(checkpoint(homeDir, { dispositions })).rejects.toThrow(); + }); + + const memoryRevision = (await git(sites, ['rev-parse', 'HEAD'])).trim(); + const pendingBody = JSON.parse(await readProductFile('example.test', `candidates/${second.id}.json`, { homeDir }) ?? ''); + pendingBody.status = 'pending'; + pendingBody.evidence_role = null; + pendingBody.memory_commit = null; + pendingBody.reviewed_at = null; + pendingBody.rejection_reason = null; + await writeProductFile('example.test', `candidates/${second.id}.json`, `${JSON.stringify(pendingBody, null, 2)}\n`, { homeDir }); + + const resumed = await checkpoint(homeDir, { expectedRevision: memoryRevision, dispositions }); + expect(resumed.status).toBe('committed'); + if (resumed.status !== 'committed') throw new Error('expected commit'); + expect(resumed.memoryCommit).toBe(memoryRevision); + expect(resumed.provenanceCommit).toMatch(/^[0-9a-f]{40}$/); + expect((await git(sites, ['log', '--oneline', '--', 'example.test/sitemap/SITE.md'])).trim().split('\n')).toHaveLength(2); + expect((await showCandidate('example.test', first.id, { homeDir })).status).toBe('ingested'); + expect((await showCandidate('example.test', second.id, { homeDir }))).toMatchObject({ + status: 'ingested', + memoryCommit: memoryRevision, + }); + }); + + it('rejects provenance recovery when a candidate does not match the requested terminal state', async () => { + const { homeDir, sites } = await primed(); + const first = await addCandidate(candidate(homeDir, { observedAt: '2026-08-30T12:00:00Z' })); + const second = await addCandidate(candidate(homeDir, { observedAt: '2026-08-31T12:00:00Z', claim: 'Later' })); + const next = `# Example\n\n${FACT}- [verified 2026-08-31] Later path is denser.\n`; + await writeDraft(homeDir, { 'sitemap/SITE.md': next }); + const dispositions = [ + { id: first.id, status: 'ingested' as const, evidenceRole: 'supporting' as const }, + { id: second.id, status: 'ingested' as const, evidenceRole: 'supporting' as const }, + ]; + + await withGitWrapper(homeDir, 'provenance', async () => { + await expect(checkpoint(homeDir, { dispositions })).rejects.toThrow(); + }); + + const memoryRevision = (await git(sites, ['rev-parse', 'HEAD'])).trim(); + const mismatched = JSON.parse(await readProductFile('example.test', `candidates/${first.id}.json`, { homeDir }) ?? ''); + mismatched.status = 'rejected'; + mismatched.evidence_role = null; + mismatched.memory_commit = null; + mismatched.rejection_reason = 'stale'; + await writeProductFile('example.test', `candidates/${first.id}.json`, `${JSON.stringify(mismatched, null, 2)}\n`, { homeDir }); + + await expect(checkpoint(homeDir, { expectedRevision: memoryRevision, dispositions })).rejects.toThrow(/pending|transition|status|mismatch/i); + expect((await git(sites, ['log', '--oneline', '--', 'example.test/sitemap/SITE.md'])).trim().split('\n')).toHaveLength(2); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBe(next); + }); }); describe('checkpoint rewrite bounds', () => { @@ -410,7 +614,7 @@ async function writeDraft(homeDir: string, files: Record, taskId } } -async function withGitWrapper(homeDir: string, fail: 'memory' | 'provenance', fn: () => Promise) { +async function withGitWrapper(homeDir: string, fail: 'memory' | 'provenance', fn: () => Promise, chmodOnFail?: string) { const originalPath = process.env.PATH; const wrapperDir = await mkdtemp(join(tmpdir(), 'webcmd-checkpoint-git-')); tempHomes.push(wrapperDir); @@ -418,10 +622,14 @@ async function withGitWrapper(homeDir: string, fail: 'memory' | 'provenance', fn const needle = fail === 'memory' ? 'checkpoint memory' : 'checkpoint provenance'; const wrapper = join(wrapperDir, 'git'); await writeFile(wrapper, `#!/usr/bin/env node +const { chmodSync } = require('node:fs'); const { spawnSync } = require('node:child_process'); const args = process.argv.slice(2); const msg = args.includes('-m') ? args[args.indexOf('-m') + 1] : ''; -if (args.includes('commit') && msg.includes(${JSON.stringify(needle)})) process.exit(1); +if (args.includes('commit') && msg.includes(${JSON.stringify(needle)})) { + ${chmodOnFail ? `try { chmodSync(${JSON.stringify(chmodOnFail)}, 0o555); } catch {}` : ''} + process.exit(1); +} const result = spawnSync(${JSON.stringify(stdout.trim())}, args, { stdio: 'inherit' }); process.exit(result.status ?? 1); `); diff --git a/src/site-memory/checkpoint.ts b/src/site-memory/checkpoint.ts index 552c3f09..c2aba175 100644 --- a/src/site-memory/checkpoint.ts +++ b/src/site-memory/checkpoint.ts @@ -1,8 +1,9 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { readCandidateRecord, updateCandidateRecord } from './candidates.js'; +import { parseProductManifest } from './context.js'; import { openSitesRepository } from './git-store.js'; -import { containedRelativePath, copyDraftFiles, readProductFile, sitesRoot, type LocalStoreOptions } from './local-store.js'; +import { containedRelativePath, copyDraftFiles, deleteProductFile, readProductFile, sitesRoot, writeProductFile, type LocalStoreOptions } from './local-store.js'; import type { Candidate, CandidateDisposition, @@ -27,17 +28,17 @@ const POINTER = /\]\(references\/[^)]+\)/; export async function checkpointMemory(input: CheckpointInput): Promise { if (!REASONS.has(input.reason)) throw new Error(`Invalid checkpoint reason: ${input.reason}`); - const paths = input.paths.map(assertMarkdownPath); + const paths = unique(input.paths.map(assertMarkdownPath), 'Duplicate site memory Markdown path.'); const product = canonicalProductKey(input.product).key; const taskId = memorySegment(input.taskId); - const dispositions = input.dispositions ?? []; + const dispositions = uniqueBy(input.dispositions ?? [], (row) => row.id, 'Duplicate candidate id.'); const repo = await openSitesRepository(input); return repo.withRepositoryLock(async () => { const actual = await repo.revision(); const loaded = await Promise.all(dispositions.map((row) => readCandidateRecord(product, row.id, input))); - if (actual && input.expectedRevision === actual && isIncompleteProvenance(loaded, dispositions, actual)) { + if (actual && input.expectedRevision === actual && isProvenanceRecovery(loaded, dispositions, actual)) { await writeDispositions(product, loaded, dispositions, actual, input); const provenanceCommit = await repo.commit( dispositions.map((row) => `${product}/candidates/${row.id}.json`), @@ -50,23 +51,42 @@ export async function checkpointMemory(input: CheckpointInput): Promise 0) { + throw new Error('Checkpoint reason rejects dispositions.'); + } validateDispositions(loaded, dispositions); + const prior = new Map(); const changed: string[] = []; for (const path of paths) { - if (drafts.get(path) !== await readProductFile(product, path, input)) changed.push(path); + const current = await readProductFile(product, path, input); + prior.set(path, current); + if (drafts.get(path) !== current) changed.push(path); + } + if (input.reason === 'candidate_ingestion' && changed.length === 0) { + throw new Error('candidate_ingestion requires a memory change.'); } let memoryCommit = actual; if (changed.length > 0) { - await copyDraftFiles(product, taskId, paths, input); - memoryCommit = await repo.commit(paths.map((path) => `${product}/${path}`), `checkpoint memory ${product}`); + try { + await copyDraftFiles(product, taskId, paths, input); + memoryCommit = await repo.commit(paths.map((path) => `${product}/${path}`), `checkpoint memory ${product}`); + } catch (err) { + const rollback = await restoreCopiedMarkdown(product, prior, input); + if (rollback.length > 0) { + throw new Error([err, ...rollback].map(errorMessage).join('; ')); + } + throw err; + } } if (!memoryCommit) throw new Error('Refusing to checkpoint without a memory revision.'); @@ -81,18 +101,44 @@ export async function checkpointMemory(input: CheckpointInput): Promise { + let seenTerminal = false; + for (const [index, row] of dispositions.entries()) { const candidate = loaded[index]; - if (candidate.status !== row.status) return false; - if (row.status === 'ingested') return candidate.memoryCommit === actual; - return candidate.reviewedAt !== null && candidate.rejectionReason !== null; - }); + if (candidate.status === 'pending') continue; + if (!matchesRequestedTerminal(candidate, row, actual)) throw new Error('Invalid candidate status transition.'); + seenTerminal = true; + } + return seenTerminal; +} + +function matchesRequestedTerminal( + candidate: Candidate, + row: CandidateDisposition, + actual: MemoryRevision, +): boolean { + if (row.status === 'ingested') { + return candidate.status === 'ingested' + && candidate.memoryCommit === actual + && (row.evidenceRole === 'supporting' || row.evidenceRole === 'dissenting' + ? candidate.evidenceRole === row.evidenceRole + : candidate.evidenceRole === 'supporting' || candidate.evidenceRole === 'dissenting') + && candidate.rejectionReason === null; + } + if (row.status === 'rejected') { + return candidate.status === 'rejected' + && candidate.memoryCommit === null + && candidate.evidenceRole === null + && candidate.reviewedAt !== null + && candidate.rejectionReason !== null + && (!row.rejectionReason || candidate.rejectionReason === row.rejectionReason); + } + return false; } async function writeDispositions( @@ -105,6 +151,7 @@ async function writeDispositions( const reviewedAt = new Date().toISOString(); for (const [index, row] of dispositions.entries()) { const current = loaded[index]; + if (current.status !== 'pending') continue; const next: Candidate = row.status === 'ingested' ? { ...current, @@ -211,16 +258,34 @@ function assertMarkdownPath(path: string): string { throw new Error(`Invalid site memory Markdown path: ${path}`); } -async function isLegacy(product: string, opts: LocalStoreOptions): Promise { - if (!await readProductFile(product, 'sitemap/SITE.md', opts)) return false; - const raw = await readProductFile(product, 'manifest.json', opts); - if (!raw) return true; - try { - const value = JSON.parse(raw) as { schemaVersion?: unknown }; - return value.schemaVersion !== 1; - } catch { - return true; +async function restoreCopiedMarkdown( + product: string, + prior: Map, + opts: LocalStoreOptions, +): Promise { + const errors: unknown[] = []; + for (const [path, body] of prior) { + try { + if (body === null) await deleteProductFile(product, path, opts); + else await writeProductFile(product, path, body, opts); + } catch (err) { + errors.push(err); + } } + return errors; +} + +function unique(values: T[], message: string): T[] { + if (new Set(values).size !== values.length) throw new Error(message); + return values; +} + +function uniqueBy(values: T[], key: (value: T) => string, message: string): T[] { + return unique(values.map(key), message) && values; +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); } async function readDrafts( diff --git a/src/site-memory/context.test.ts b/src/site-memory/context.test.ts index 1bde5308..a7f74f6b 100644 --- a/src/site-memory/context.test.ts +++ b/src/site-memory/context.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { getMemoryContext } from './context.js'; +import { getMemoryContext, parseProductManifest } from './context.js'; import { openSitesRepository } from './git-store.js'; import { readProductFile, writeProductFile } from './local-store.js'; import type { GlobalSeedProvider } from './seed-client.js'; @@ -17,6 +17,24 @@ afterEach(async () => { await Promise.all(tempHomes.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); }); +describe('parseProductManifest', () => { + const product = { + key: 'example.test', + hostname: 'example.test', + displayHostname: 'example.test', + registrableDomain: 'example.test', + }; + const valid = { schemaVersion: 1, product, interfaces: [], seed: { status: 'absent' as const } }; + + it('accepts a v1 manifest and rejects missing, partial, malformed, and non-v1 input', () => { + expect(parseProductManifest(`${JSON.stringify(valid)}\n`)).toEqual(valid); + expect(parseProductManifest(null)).toBeUndefined(); + expect(parseProductManifest('{')).toBeUndefined(); + expect(parseProductManifest(JSON.stringify({ ...valid, schemaVersion: 2 }))).toBeUndefined(); + expect(parseProductManifest(JSON.stringify({ schemaVersion: 1, product, seed: { status: 'absent' } }))).toBeUndefined(); + }); +}); + describe('memory context initialization', () => { it('persists a terminal absent result without creating SITE.md', async () => { const { homeDir, sites } = await tempSites(); diff --git a/src/site-memory/context.ts b/src/site-memory/context.ts index 362de94d..b7ca2e6b 100644 --- a/src/site-memory/context.ts +++ b/src/site-memory/context.ts @@ -73,7 +73,7 @@ async function persistSeed( opts: LocalStoreOptions, ): Promise { await repo.withRepositoryLock(async () => { - if (parseManifest(await readProductFile(product.key, 'manifest.json', opts))) return; + if (parseProductManifest(await readProductFile(product.key, 'manifest.json', opts))) return; const persisted: PersistedSeedResult = seed.status === 'available' ? { status: 'available', revision: seed.revision } : seed; const manifest: ProductManifest = { schemaVersion: 1, product, interfaces: [], seed: persisted }; const files = ['manifest.json']; @@ -133,13 +133,13 @@ async function writeContained(root: string, path: string, body: string): Promise async function loadManifests(opts: LocalStoreOptions): Promise { const manifests: ProductManifest[] = []; for (const key of await listProductKeys(opts)) { - const parsed = parseManifest(await readProductFile(key, 'manifest.json', opts)); + const parsed = parseProductManifest(await readProductFile(key, 'manifest.json', opts)); if (parsed) manifests.push(parsed); } return manifests; } -function parseManifest(raw: string | null): ProductManifest | undefined { +export function parseProductManifest(raw: string | null): ProductManifest | undefined { if (!raw) return undefined; try { const value = JSON.parse(raw) as unknown; @@ -176,7 +176,7 @@ function isPersistedSeed(value: unknown): value is PersistedSeedResult { async function isLegacySite(productKey: string, opts: LocalStoreOptions): Promise { const site = await readProductFile(productKey, 'sitemap/SITE.md', opts); if (!site) return false; - return !parseManifest(await readProductFile(productKey, 'manifest.json', opts)); + return !parseProductManifest(await readProductFile(productKey, 'manifest.json', opts)); } async function listReferences(productKey: string, opts: LocalStoreOptions): Promise<{ path: string }[]> { diff --git a/src/site-memory/local-store.test.ts b/src/site-memory/local-store.test.ts index 633c888b..9e3a18b4 100644 --- a/src/site-memory/local-store.test.ts +++ b/src/site-memory/local-store.test.ts @@ -12,6 +12,7 @@ import { addResponseSample, appendNote, copyDraftFiles, + deleteProductFile, getVerifyFixture, listProductKeys, listSiteMemory, @@ -282,6 +283,18 @@ describe('local site memory store', () => { .rejects.toThrow(/Invalid site memory path/); }); + it('deletes a contained product file and ignores a missing path', async () => { + const homeDir = await tempHome(); + await writeProductFile('example.test', 'sitemap/SITE.md', '# Example\n', { homeDir }); + + await deleteProductFile('example.test', 'sitemap/SITE.md', { homeDir }); + + await expect(readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).resolves.toBeNull(); + await expect(deleteProductFile('example.test', 'sitemap/SITE.md', { homeDir })).resolves.toBeUndefined(); + await expect(deleteProductFile('example.test', '../outside.md', { homeDir })) + .rejects.toThrow(/Invalid site memory path/); + }); + it('copies contained draft files into product memory', async () => { const homeDir = await tempHome(); const draft = join(homeDir, '.webcmd/sites/.drafts/task-1/example.test/sitemap/SITE.md'); diff --git a/src/site-memory/local-store.ts b/src/site-memory/local-store.ts index 322977a6..3f0ea009 100644 --- a/src/site-memory/local-store.ts +++ b/src/site-memory/local-store.ts @@ -197,6 +197,17 @@ export async function writeProductFile(productKey: string, path: string, body: s await withWriteLock(target, () => atomicWrite(target, body)); } +export async function deleteProductFile(productKey: string, path: string, opts: LocalStoreOptions = {}): Promise { + const productRoot = join(sitesRoot(opts), productSegment(productKey)); + const relative = containedRelativePath(productRoot, path); + try { + await unlink(join(productRoot, ...relative.split('/'))); + } catch (err) { + if (isNodeError(err) && err.code === 'ENOENT') return; + throw err; + } +} + export async function copyDraftFiles(productKey: string, taskId: string, paths: string[], opts: LocalStoreOptions = {}): Promise { const draftRoot = join(sitesRoot(opts), '.drafts', productSegment(taskId), productSegment(productKey)); for (const path of paths) { From 92d88ed1e518de4d77fae3c308fdda931fd65221 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Tue, 1 Sep 2026 01:07:02 +0530 Subject: [PATCH 14/41] fix(site-memory): validate recovery and rejection-only ingestion --- src/site-memory/checkpoint.test.ts | 132 ++++++++++++++++++++++++++++- src/site-memory/checkpoint.ts | 70 +++++++-------- src/site-memory/git-store.test.ts | 16 ++++ src/site-memory/git-store.ts | 9 ++ 4 files changed, 192 insertions(+), 35 deletions(-) diff --git a/src/site-memory/checkpoint.test.ts b/src/site-memory/checkpoint.test.ts index a1d33ba8..90b3c474 100644 --- a/src/site-memory/checkpoint.test.ts +++ b/src/site-memory/checkpoint.test.ts @@ -85,6 +85,12 @@ describe('checkpoint compare-and-swap', () => { product: { key: 'example.test', hostname: 'example.test', displayHostname: 'example.test', registrableDomain: 'example.test' }, seed: { status: 'absent' }, })], + ['mismatched key', JSON.stringify({ + schemaVersion: 1, + product: { key: 'other.test', hostname: 'other.test', displayHostname: 'other.test', registrableDomain: 'other.test' }, + interfaces: [], + seed: { status: 'absent' }, + })], ] as const)('refuses a %s product manifest without copying the draft', async (_label, body) => { const { homeDir, sites, revision: initial } = await primed(); const manifest = join(sites, 'example.test', 'manifest.json'); @@ -217,6 +223,39 @@ describe('checkpoint candidate dispositions', () => { expect((await showCandidate('example.test', first.id, { homeDir })).status).toBe('pending'); }); + it('commits provenance only when candidate_ingestion rejects without a memory change', async () => { + const { homeDir, sites } = await primed(); + const pending = await addCandidate(candidate(homeDir)); + const revision = (await git(sites, ['rev-parse', 'HEAD'])).trim(); + const next = `# Example\n\n${FACT}- [verified 2026-08-31] Later path is denser.\n`; + await writeDraft(homeDir, { 'sitemap/SITE.md': next }); + await expect(checkpoint(homeDir, { + dispositions: [{ id: pending.id, status: 'rejected', rejectionReason: 'stale' }], + })).rejects.toThrow(/unchanged|memory|change/i); + expect((await showCandidate('example.test', pending.id, { homeDir })).status).toBe('pending'); + + await writeDraft(homeDir, { 'sitemap/SITE.md': SITE }); + const result = await checkpoint(homeDir, { + dispositions: [{ id: pending.id, status: 'rejected', rejectionReason: 'stale' }], + }); + expect(result.status).toBe('committed'); + if (result.status !== 'committed') throw new Error('expected commit'); + expect(result.memoryCommit).toBe(revision); + expect(result.provenanceCommit).toMatch(/^[0-9a-f]{40}$/); + expect(result.provenanceCommit).not.toBe(revision); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBe(SITE); + expect((await git(sites, ['log', '--oneline', '--', 'example.test/sitemap/SITE.md'])).trim().split('\n')).toHaveLength(1); + expect((await showCandidate('example.test', pending.id, { homeDir }))).toMatchObject({ + status: 'rejected', + evidenceRole: null, + memoryCommit: null, + rejectionReason: 'stale', + }); + const provenanceFiles = (await git(sites, ['show', '--name-only', '--pretty=format:', result.provenanceCommit ?? ''])).trim().split('\n'); + expect(provenanceFiles.join('\n')).toMatch(/candidates/); + expect(provenanceFiles.join('\n')).not.toMatch(/SITE\.md/); + }); + it('rejects dispositions on direct_correction and major_rewrite', async () => { const { homeDir } = await primed(); const pending = await addCandidate(candidate(homeDir)); @@ -374,12 +413,27 @@ describe('checkpoint git transaction', () => { try { await withGitWrapper(blocked.homeDir, 'memory', async () => { - await expect(checkpoint(blocked.homeDir, { + const error = await checkpoint(blocked.homeDir, { dispositions: [ { id: pending.id, status: 'ingested', evidenceRole: 'supporting' }, { id: later.id, status: 'ingested', evidenceRole: 'supporting' }, ], - })).rejects.toThrow(/checkpoint memory[\s\S]*(EACCES|EPERM|permission denied)|(EACCES|EPERM|permission denied)[\s\S]*checkpoint memory/i); + }).then( + () => { + throw new Error('expected checkpoint to fail'); + }, + (err: unknown) => err, + ); + expect(error).toBeInstanceOf(AggregateError); + const aggregate = error as AggregateError; + expect(aggregate.errors.length).toBeGreaterThanOrEqual(2); + expect(aggregate.message).toMatch(/rollback/i); + expect(aggregate.errors.map((item) => (item instanceof Error ? item.message : String(item))).join('\n')).toMatch( + /checkpoint memory/i, + ); + expect(aggregate.errors.map((item) => (item instanceof Error ? item.message : String(item))).join('\n')).toMatch( + /EACCES|EPERM|permission denied/i, + ); }, sitemapDir); } finally { await chmod(sitemapDir, 0o755).catch(() => undefined); @@ -482,6 +536,80 @@ describe('checkpoint git transaction', () => { expect((await git(sites, ['log', '--oneline', '--', 'example.test/sitemap/SITE.md'])).trim().split('\n')).toHaveLength(2); expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBe(next); }); + + it('validates the product manifest before provenance recovery', async () => { + const { homeDir, sites } = await primed(); + const first = await addCandidate(candidate(homeDir, { observedAt: '2026-08-30T12:00:00Z' })); + const second = await addCandidate(candidate(homeDir, { observedAt: '2026-08-31T12:00:00Z', claim: 'Later' })); + const next = `# Example\n\n${FACT}- [verified 2026-08-31] Later path is denser.\n`; + await writeDraft(homeDir, { 'sitemap/SITE.md': next }); + const dispositions = [ + { id: first.id, status: 'ingested' as const, evidenceRole: 'supporting' as const }, + { id: second.id, status: 'ingested' as const, evidenceRole: 'supporting' as const }, + ]; + + await withGitWrapper(homeDir, 'provenance', async () => { + await expect(checkpoint(homeDir, { dispositions })).rejects.toThrow(); + }); + + const memoryRevision = (await git(sites, ['rev-parse', 'HEAD'])).trim(); + await writeProductFile('example.test', 'manifest.json', '{\n', { homeDir }); + + await expect(checkpoint(homeDir, { expectedRevision: memoryRevision, dispositions })).rejects.toThrow(/incompatible beta schema|manifest|schema/i); + expect(await readProductFile('example.test', 'sitemap/SITE.md', { homeDir })).toBe(next); + expect((await git(sites, ['log', '--oneline', '--', 'example.test/sitemap/SITE.md'])).trim().split('\n')).toHaveLength(2); + expect(JSON.parse(await git(sites, ['show', `HEAD:example.test/candidates/${first.id}.json`])).status).toBe('pending'); + }); + + it('validates disposition fields during provenance recovery', async () => { + const { homeDir, sites } = await primed(); + const first = await addCandidate(candidate(homeDir, { observedAt: '2026-08-30T12:00:00Z' })); + const second = await addCandidate(candidate(homeDir, { observedAt: '2026-08-31T12:00:00Z', claim: 'Later' })); + const next = `# Example\n\n${FACT}- [verified 2026-08-31] Later path is denser.\n`; + await writeDraft(homeDir, { 'sitemap/SITE.md': next }); + const dispositions = [ + { id: first.id, status: 'ingested' as const, evidenceRole: 'supporting' as const }, + { id: second.id, status: 'ingested' as const, evidenceRole: 'supporting' as const }, + ]; + + await withGitWrapper(homeDir, 'provenance', async () => { + await expect(checkpoint(homeDir, { dispositions })).rejects.toThrow(); + }); + + const memoryRevision = (await git(sites, ['rev-parse', 'HEAD'])).trim(); + await expect(checkpoint(homeDir, { + expectedRevision: memoryRevision, + dispositions: [ + { id: first.id, status: 'ingested' }, + { id: second.id, status: 'ingested' }, + ], + })).rejects.toThrow(/evidence_role|status/i); + expect((await git(sites, ['log', '--oneline', '--', 'example.test/sitemap/SITE.md'])).trim().split('\n')).toHaveLength(2); + expect(JSON.parse(await git(sites, ['show', `HEAD:example.test/candidates/${first.id}.json`])).status).toBe('pending'); + }); + + it('rejects provenance recovery when candidate paths match HEAD', async () => { + const { homeDir, sites } = await primed(); + const first = await addCandidate(candidate(homeDir, { observedAt: '2026-08-30T12:00:00Z' })); + const second = await addCandidate(candidate(homeDir, { observedAt: '2026-08-31T12:00:00Z', claim: 'Later' })); + const next = `# Example\n\n${FACT}- [verified 2026-08-31] Later path is denser.\n`; + await writeDraft(homeDir, { 'sitemap/SITE.md': next }); + const dispositions = [ + { id: first.id, status: 'ingested' as const, evidenceRole: 'supporting' as const }, + { id: second.id, status: 'ingested' as const, evidenceRole: 'supporting' as const }, + ]; + + await withGitWrapper(homeDir, 'provenance', async () => { + await expect(checkpoint(homeDir, { dispositions })).rejects.toThrow(); + }); + + await git(sites, ['add', '--', `example.test/candidates/${first.id}.json`, `example.test/candidates/${second.id}.json`]); + await git(sites, ['-c', 'user.name=webcmd', '-c', 'user.email=webcmd@local', 'commit', '-m', 'manual-provenance']); + const head = (await git(sites, ['rev-parse', 'HEAD'])).trim(); + + await expect(checkpoint(homeDir, { expectedRevision: head, dispositions })).rejects.toThrow(/pending|transition|status/i); + expect((await git(sites, ['rev-parse', 'HEAD'])).trim()).toBe(head); + }); }); describe('checkpoint rewrite bounds', () => { diff --git a/src/site-memory/checkpoint.ts b/src/site-memory/checkpoint.ts index c2aba175..7b795417 100644 --- a/src/site-memory/checkpoint.ts +++ b/src/site-memory/checkpoint.ts @@ -36,14 +36,22 @@ export async function checkpointMemory(input: CheckpointInput): Promise { const actual = await repo.revision(); + const manifest = parseProductManifest(await readProductFile(product, 'manifest.json', input)); + if (!manifest || manifest.product.key !== product) { + throw new Error('Incompatible beta schema; learning is read-only until this SITE.md is cleared.'); + } const loaded = await Promise.all(dispositions.map((row) => readCandidateRecord(product, row.id, input))); + for (const row of dispositions) validateDispositionFields(row); + const candidatePaths = dispositions.map((row) => `${product}/candidates/${row.id}.json`); - if (actual && input.expectedRevision === actual && isProvenanceRecovery(loaded, dispositions, actual)) { + if ( + actual + && input.expectedRevision === actual + && isProvenanceRecovery(loaded, dispositions, actual) + && await repo.pathsChanged(candidatePaths) + ) { await writeDispositions(product, loaded, dispositions, actual, input); - const provenanceCommit = await repo.commit( - dispositions.map((row) => `${product}/candidates/${row.id}.json`), - `checkpoint provenance ${product}`, - ); + const provenanceCommit = await repo.commit(candidatePaths, `checkpoint provenance ${product}`); return { status: 'committed', memoryCommit: actual, provenanceCommit }; } @@ -51,10 +59,6 @@ export async function checkpointMemory(input: CheckpointInput): Promise row.status === 'ingested'); + if (ingested && changed.length === 0) throw new Error('candidate_ingestion requires a memory change.'); + if (!ingested && changed.length > 0) throw new Error('candidate_ingestion rejections require unchanged memory.'); } let memoryCommit = actual; if (changed.length > 0) { @@ -83,7 +89,7 @@ export async function checkpointMemory(input: CheckpointInput): Promise 0) { - throw new Error([err, ...rollback].map(errorMessage).join('; ')); + throw new AggregateError([err, ...rollback], 'Memory commit failed and rollback also failed'); } throw err; } @@ -125,9 +131,7 @@ function matchesRequestedTerminal( if (row.status === 'ingested') { return candidate.status === 'ingested' && candidate.memoryCommit === actual - && (row.evidenceRole === 'supporting' || row.evidenceRole === 'dissenting' - ? candidate.evidenceRole === row.evidenceRole - : candidate.evidenceRole === 'supporting' || candidate.evidenceRole === 'dissenting') + && candidate.evidenceRole === row.evidenceRole && candidate.rejectionReason === null; } if (row.status === 'rejected') { @@ -173,20 +177,23 @@ async function writeDispositions( } } -function validateDispositions(loaded: Candidate[], dispositions: CandidateDisposition[]): void { - for (const [index, row] of dispositions.entries()) { - if (loaded[index].status !== 'pending') throw new Error('Invalid candidate status transition.'); - if (row.status === 'ingested') { - if (row.evidenceRole !== 'supporting' && row.evidenceRole !== 'dissenting') { - throw new Error('Invalid candidate evidence_role.'); - } - if (row.rejectionReason) throw new Error('Invalid candidate status.'); - } else if (row.status === 'rejected') { - if (!row.rejectionReason?.trim()) throw new Error('Rejected candidates require a reason.'); - if (row.evidenceRole) throw new Error('Invalid candidate status.'); - } else { - throw new Error('Invalid candidate status.'); +function validateDispositionFields(row: CandidateDisposition): void { + if (row.status === 'ingested') { + if (row.evidenceRole !== 'supporting' && row.evidenceRole !== 'dissenting') { + throw new Error('Invalid candidate evidence_role.'); } + if (row.rejectionReason) throw new Error('Invalid candidate status.'); + } else if (row.status === 'rejected') { + if (!row.rejectionReason?.trim()) throw new Error('Rejected candidates require a reason.'); + if (row.evidenceRole) throw new Error('Invalid candidate status.'); + } else { + throw new Error('Invalid candidate status.'); + } +} + +function validateDispositions(loaded: Candidate[], dispositions: CandidateDisposition[]): void { + for (const candidate of loaded) { + if (candidate.status !== 'pending') throw new Error('Invalid candidate status transition.'); } const ingested = dispositions.map((row, index) => ({ row, candidate: loaded[index] })).filter((entry) => entry.row.status === 'ingested'); if (ingested.length === 0) return; @@ -281,11 +288,8 @@ function unique(values: T[], message: string): T[] { } function uniqueBy(values: T[], key: (value: T) => string, message: string): T[] { - return unique(values.map(key), message) && values; -} - -function errorMessage(err: unknown): string { - return err instanceof Error ? err.message : String(err); + unique(values.map(key), message); + return values; } async function readDrafts( diff --git a/src/site-memory/git-store.test.ts b/src/site-memory/git-store.test.ts index faa1f40f..9a7b35ba 100644 --- a/src/site-memory/git-store.test.ts +++ b/src/site-memory/git-store.test.ts @@ -190,6 +190,22 @@ describe('sites git repository', () => { expect(await git(sites, ['status', '--porcelain', '-uall'])).not.toMatch(/^(A |AD)/m); }); + it('reports whether explicit paths differ from HEAD', async () => { + const { homeDir } = await tempSites(); + await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); + const repo = await openSitesRepository({ homeDir }); + await repo.commit(['example.test/manifest.json'], 'init'); + + expect(await repo.pathsChanged(['example.test/manifest.json'])).toBe(false); + + await writeProductFile('example.test', 'manifest.json', '{"dirty":true}\n', { homeDir }); + expect(await repo.pathsChanged(['example.test/manifest.json'])).toBe(true); + + await writeProductFile('example.test', 'notes.md', 'untracked\n', { homeDir }); + expect(await repo.pathsChanged(['example.test/notes.md'])).toBe(true); + expect(await repo.pathsChanged(['example.test/missing.md'])).toBe(false); + }); + it('exposes commit and cleanup failures together when restore also fails', async () => { const { homeDir } = await tempSites(); await writeProductFile('example.test', 'manifest.json', '{}\n', { homeDir }); diff --git a/src/site-memory/git-store.ts b/src/site-memory/git-store.ts index 2944e8a1..e2dbaefd 100644 --- a/src/site-memory/git-store.ts +++ b/src/site-memory/git-store.ts @@ -20,6 +20,7 @@ const GIT_FLAGS = [ export interface SitesRepository { revision(): Promise; commit(paths: string[], message: string): Promise; + pathsChanged(paths: string[]): Promise; withRepositoryLock(fn: () => Promise): Promise; } @@ -29,6 +30,7 @@ export async function openSitesRepository(options: LocalStoreOptions = {}): Prom return { revision: () => revisionOf(root), commit: (paths, message) => withRepositoryLock(root, () => commitPaths(root, paths, message)), + pathsChanged: (paths) => pathsDifferFromHead(root, paths), withRepositoryLock: (fn) => withRepositoryLock(root, fn), }; } @@ -66,6 +68,13 @@ async function commitPaths(root: string, paths: string[], message: string): Prom return (await git(root, ['rev-parse', 'HEAD'])).trim(); } +async function pathsDifferFromHead(root: string, paths: string[]): Promise { + if (paths.length === 0) return false; + const relativePaths = paths.map((path) => containedRelativePath(root, path)); + const status = await git(root, ['status', '--porcelain', '-uall', '--', ...relativePaths]); + return status.split('\n').some(Boolean); +} + async function restoreStagedPaths(root: string, relativePaths: string[]): Promise { const paths = [...relativePaths, '.gitignore']; if (await revisionOf(root) !== null) { From c5f6ec0617c1b2dfc0085b9b6dd0b14c4ed68a4f Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Tue, 1 Sep 2026 01:18:36 +0530 Subject: [PATCH 15/41] feat(site-memory): expose the learning workflow --- src/cli.test.ts | 80 ++++++------ src/cli.ts | 141 +++------------------ src/hosted/runner.test.ts | 18 +++ src/site-memory/commands.test.ts | 199 ++++++++++++++++++++++++++++- src/site-memory/commands.ts | 206 +++++++++++++++++++++++++++++++ 5 files changed, 482 insertions(+), 162 deletions(-) diff --git a/src/cli.test.ts b/src/cli.test.ts index f138106e..4000b1b8 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -1550,66 +1550,56 @@ describe('selectFreshByTimestamp', () => { }); describe('resolveSitemapAvailabilityForUrl', () => { - function registryFor(site: string, domain: string): Map { - return new Map([[`${site}:read`, { - site, - name: 'read', - access: 'read', - description: 'read', - domain, - browser: false, - args: [], - }]]); - } - - it('detects local sitemap overlays using adapter registry domain matches', () => { + it('resolves the product key and local SITE.md without registry names', () => { const homeDir = path.join(os.tmpdir(), 'webcmd-sitemap-home'); - const packageRoot = path.join(os.tmpdir(), 'webcmd-sitemap-package'); - const localSitemap = path.join(homeDir, '.webcmd', 'sites', 'hackernews', 'sitemap'); - const exists = new Set([localSitemap]); + const localSitemap = path.join(homeDir, '.webcmd', 'sites', 'news.ycombinator.com', 'sitemap', 'SITE.md'); const report = resolveSitemapAvailabilityForUrl('https://news.ycombinator.com/item?id=1', { homeDir, - packageRoot, - registry: registryFor('hackernews', 'news.ycombinator.com'), - fileExists: (candidate) => exists.has(candidate), + fileExists: (candidate) => candidate === localSitemap, }); expect(report).toMatchObject({ - site: 'hackernews', + site: 'news.ycombinator.com', available: true, source: 'local', paths: { local: localSitemap }, }); - expect(report?.hint).toContain('webcmd-browser-sitemap'); + expect(report?.hint).toContain('site memory context'); + expect(JSON.stringify(report)).not.toMatch(/local\+global|hackernews|webcmd-browser-sitemap/); }); - it('reports global+local when both sitemap layers exist', () => { + it('ignores package sitemaps and registry aliases', () => { const homeDir = path.join(os.tmpdir(), 'webcmd-sitemap-home'); const packageRoot = path.join(os.tmpdir(), 'webcmd-sitemap-package'); - const localSitemap = path.join(homeDir, '.webcmd', 'sites', 'twitter', 'sitemap.md'); - const globalSitemap = path.join(packageRoot, 'sitemaps', 'twitter'); - const exists = new Set([localSitemap, globalSitemap]); + const packageSitemap = path.join(packageRoot, 'sitemaps', 'twitter'); + const aliasSitemap = path.join(homeDir, '.webcmd', 'sites', 'twitter', 'sitemap.md'); const report = resolveSitemapAvailabilityForUrl('https://x.com/webcmd', { homeDir, - packageRoot, - registry: registryFor('twitter', 'x.com'), - fileExists: (candidate) => exists.has(candidate), + fileExists: (candidate) => candidate === packageSitemap || candidate === aliasSitemap, }); - expect(report).toMatchObject({ - site: 'twitter', - source: 'local+global', - paths: { local: localSitemap, global: globalSitemap }, - }); + expect(report).toBeNull(); }); - it('returns null when no sitemap layer exists', () => { + it('returns availability for existing SITE.md even when learning is read-only', () => { + const homeDir = path.join(os.tmpdir(), 'webcmd-sitemap-home'); + const localSitemap = path.join(homeDir, '.webcmd', 'sites', 'example.test', 'sitemap', 'SITE.md'); + + expect(() => resolveSitemapAvailabilityForUrl('https://example.test/', { + homeDir, + fileExists: (candidate) => candidate === localSitemap, + })).not.toThrow(); + expect(resolveSitemapAvailabilityForUrl('https://example.test/', { + homeDir, + fileExists: (candidate) => candidate === localSitemap, + })?.available).toBe(true); + }); + + it('returns null when no local SITE.md exists', () => { const report = resolveSitemapAvailabilityForUrl('https://example.com/', { homeDir: path.join(os.tmpdir(), 'webcmd-sitemap-home'), - packageRoot: path.join(os.tmpdir(), 'webcmd-sitemap-package'), - registry: new Map(), fileExists: () => false, }); @@ -1617,6 +1607,24 @@ describe('resolveSitemapAvailabilityForUrl', () => { }); }); +describe('local learning command registration', () => { + it('registers site memory context, candidate, and checkpoint on the local program', () => { + const program = createProgram('', ''); + for (const path of [ + ['site', 'memory', 'context'], + ['site', 'memory', 'candidate', 'add'], + ['site', 'memory', 'candidate', 'search'], + ['site', 'memory', 'candidate', 'show'], + ['site', 'memory', 'candidate', 'list'], + ['site', 'memory', 'checkpoint'], + ]) { + let command: ReturnType | undefined = program; + for (const segment of path) command = command?.commands.find(child => child.name() === segment) as typeof program | undefined; + expect(command, path.join(' ')).toBeDefined(); + } + }); +}); + describe('browser verify', () => { beforeEach(() => { process.exitCode = undefined; diff --git a/src/cli.ts b/src/cli.ts index 352f6177..a1a8fcf8 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -12,7 +12,7 @@ import * as readline from 'node:readline/promises'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { Command, Option } from 'commander'; import { findPackageRoot, getBuiltEntryCandidates } from './package-paths.js'; -import { type CliCommand, getRegistry } from './registry.js'; +import { getRegistry } from './registry.js'; // Side-effect import: registers client-owned `web fetch` in the core registry // so it reaches help, `list`, completions and manifests without a plugin. import './fetch/command.js'; @@ -63,17 +63,14 @@ import { BrowserRunError } from './browser/run/types.js'; import { classifyCommandOrigin, formatCommandOrigin } from './command-origin.js'; import { readOverrideRecords, removeOverrideRecords } from './override-provenance.js'; import { clearDaemonRunContext, generateRunId, isUnknownOutcomeError, runWithDaemonRunContext } from './session-lease.js'; -import { createLocalSiteMemoryBackend, registerSiteCommands } from './site-memory/commands.js'; +import { createLocalLearningBackend, createLocalSiteMemoryBackend, registerSiteCommands } from './site-memory/commands.js'; +import { canonicalProductKey } from './site-memory/product-resolver.js'; import { resolveAdapterSourcePath, splitAdapterCommandKey } from './adapter-source.js'; const CLI_FILE = fileURLToPath(import.meta.url); const FOLLOW_POLL_MS = 1_000; const externalRootCommands = new WeakSet(); -function getBrowserCacheDir(): string { - return process.env.WEBCMD_CACHE_DIR || path.join(os.homedir(), '.webcmd', 'cache'); -} - function parsePositiveIntOption(value: string | undefined, _label: string, fallback: number): number { const parsed = value === undefined ? fallback : Number.parseInt(value, 10); return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback; @@ -329,139 +326,39 @@ export type SiteMemoryReport = { export type SitemapAvailability = { site: string; available: true; - source: 'local' | 'global' | 'local+global'; + source: 'local'; hint: string; paths: { local?: string; - global?: string; }; }; -type SitemapHintState = { - seenSites: string[]; - updatedAt: string; -}; - type SitemapAvailabilityOptions = { homeDir?: string; - packageRoot?: string; - registry?: Map; fileExists?: (candidate: string) => boolean; }; const SITEMAP_HINT = - 'Site sitemap available. For navigation context, use the webcmd-browser-sitemap skill; treat browser state as truth if it disagrees.'; + 'Product sitemap available. Use `webcmd site memory context -f json` for navigation context; treat browser state as truth if it disagrees.'; -function siteNameCandidatesFromUrl(url: string, registry: Map = getRegistry()): string[] { - let host: string; +export function resolveSitemapAvailabilityForUrl(url: string, options: SitemapAvailabilityOptions = {}): SitemapAvailability | null { + let product; try { - host = new URL(url).hostname.toLowerCase().replace(/^www\./, ''); + product = canonicalProductKey(url); } catch { - return []; - } - - const scored = new Map(); - for (const command of registry.values()) { - if (!command.domain) continue; - let domainHost = command.domain.toLowerCase().trim(); - try { - domainHost = new URL(domainHost.includes('://') ? domainHost : `https://${domainHost}`).hostname.toLowerCase(); - } catch { - domainHost = domainHost.split('/')[0] ?? domainHost; - } - domainHost = domainHost.replace(/^www\./, ''); - if (!domainHost) continue; - if (host === domainHost || host.endsWith(`.${domainHost}`)) { - scored.set(command.site, Math.max(scored.get(command.site) ?? 0, domainHost.length)); - } + return null; } - - const registrySites = [...scored.entries()] - .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) - .map(([site]) => site); - - const hostParts = host.split('.').filter(Boolean); - const fallback = hostParts.length >= 2 ? hostParts[hostParts.length - 2] : hostParts[0]; - return [...new Set([...registrySites, ...(fallback ? [fallback] : [])])]; -} - -function firstExistingSitemapPath(paths: string[], fileExists: (candidate: string) => boolean): string | undefined { - return paths.find((candidate) => fileExists(candidate)); -} - -function sitemapPathsForSite(site: string, opts: Required>): { local?: string; global?: string } { - const safeSite = site.replace(/[^a-zA-Z0-9_-]+/g, '-'); - if (!safeSite) return {}; - const localBase = path.join(opts.homeDir, '.webcmd', 'sites', safeSite); - return { - local: firstExistingSitemapPath([ - path.join(localBase, 'sitemap'), - path.join(localBase, 'sitemap.md'), - ], opts.fileExists), - global: firstExistingSitemapPath([ - path.join(opts.packageRoot, 'sitemaps', safeSite), - path.join(opts.packageRoot, 'sitemaps', `${safeSite}.md`), - ], opts.fileExists), - }; -} - -export function resolveSitemapAvailabilityForUrl(url: string, options: SitemapAvailabilityOptions = {}): SitemapAvailability | null { const homeDir = options.homeDir ?? os.homedir(); - const packageRoot = options.packageRoot ?? findPackageRoot(CLI_FILE); - const registry = options.registry ?? getRegistry(); const fileExists = options.fileExists ?? fs.existsSync; - - for (const site of siteNameCandidatesFromUrl(url, registry)) { - const paths = sitemapPathsForSite(site, { homeDir, packageRoot, fileExists }); - if (!paths.local && !paths.global) continue; - const source = paths.local && paths.global ? 'local+global' : paths.local ? 'local' : 'global'; - return { - site, - available: true, - source, - hint: SITEMAP_HINT, - paths, - }; - } - return null; -} - -function getBrowserSitemapHintStatePath(scope: string): string { - const safeScope = scope.replace(/[^a-zA-Z0-9_-]+/g, '_'); - return path.join(getBrowserCacheDir(), 'browser-sitemap-hints', `${safeScope}.json`); -} - -function loadBrowserSitemapHintState(scope: string): SitemapHintState { - try { - const parsed = JSON.parse(fs.readFileSync(getBrowserSitemapHintStatePath(scope), 'utf-8')) as SitemapHintState; - if (parsed && typeof parsed === 'object' && Array.isArray(parsed.seenSites)) { - return { - seenSites: parsed.seenSites.filter((site) => typeof site === 'string'), - updatedAt: typeof parsed.updatedAt === 'string' ? parsed.updatedAt : new Date(0).toISOString(), - }; - } - } catch { - // First command in this browser session has no hint cache yet. - } - return { seenSites: [], updatedAt: new Date(0).toISOString() }; -} - -function markBrowserSitemapHintSeen(scope: string, site: string): void { - const state = loadBrowserSitemapHintState(scope); - if (!state.seenSites.includes(site)) state.seenSites.push(site); - const target = getBrowserSitemapHintStatePath(scope); - fs.mkdirSync(path.dirname(target), { recursive: true }); - fs.writeFileSync(target, JSON.stringify({ seenSites: state.seenSites, updatedAt: new Date().toISOString() }), 'utf-8'); -} - -function sitemapHintForBrowserUrl(url: string, scope: string, opts: { oncePerSession: boolean }): SitemapAvailability | null { - const sitemap = resolveSitemapAvailabilityForUrl(url); - if (!sitemap) return null; - if (!opts.oncePerSession) return sitemap; - const state = loadBrowserSitemapHintState(scope); - if (state.seenSites.includes(sitemap.site)) return null; - markBrowserSitemapHintSeen(scope, sitemap.site); - return sitemap; + const local = path.join(homeDir, '.webcmd', 'sites', product.key, 'sitemap', 'SITE.md'); + if (!fileExists(local)) return null; + return { + site: product.key, + available: true, + source: 'local', + hint: SITEMAP_HINT, + paths: { local }, + }; } export function checkSiteMemory(site: string): SiteMemoryReport { @@ -696,7 +593,7 @@ export function createProgram(BUILTIN_CLIS: string, USER_CLIS: string, pluginsDi .name('webcmd') .description('Make any website your CLI. Zero setup. AI-powered.'); configureRootCommandSurface(program); - registerSiteCommands(program, createLocalSiteMemoryBackend()); + registerSiteCommands(program, createLocalSiteMemoryBackend(), undefined, {}, createLocalLearningBackend()); const siteCmd = program.commands.find(command => command.name() === 'site')!; // Snapshot before applyRootSubcommandSummaries() rewrites .description() to a child-name listing. const originalSiteDescription = siteCmd.description(); diff --git a/src/hosted/runner.test.ts b/src/hosted/runner.test.ts index ff1ea26f..aa266097 100644 --- a/src/hosted/runner.test.ts +++ b/src/hosted/runner.test.ts @@ -653,6 +653,24 @@ describe('runHostedCli', () => { } }); + it.each([ + ['site', 'memory', 'context', 'https://example.test/', '--task-id', 'task-1'], + ['site', 'memory', 'candidate', 'add', 'example.test', '--kind', 'access', '--claim', 'c', '--evidence', 'e', '--consequence', 'q'], + ['site', 'memory', 'checkpoint', 'example.test', '--task-id', 'task-1', '--expected-revision', 'rev', '--reason', 'direct_correction', '--paths', 'sitemap/SITE.md'], + ])('does not advertise unsupported local learning mutations on hosted: %j', async (...argv) => { + const fetchImpl = vi.fn(); + const stderr = sink(); + const result = await runHostedCli(argv, { + config: makeHostedConfig({ apiBaseUrl: 'https://api.example.com', apiKey: 'key' }), + stderr: stderr.stream, + fetchImpl, + }); + + expect(result.exitCode).toBe(2); + expect(fetchImpl).not.toHaveBeenCalled(); + expect(`${stderr.text()}${JSON.stringify(result)}`).not.toMatch(/draftPath|candidates\//); + }); + it('uses hosted manifest provenance for adapter source and keeps memory reads on stdout', async () => { const homeDir = await mkdtemp(path.join(tmpdir(), 'webcmd-hosted-authoring-')); const stdout = sink(); diff --git a/src/site-memory/commands.test.ts b/src/site-memory/commands.test.ts index 0bd4f8ab..deda2ab3 100644 --- a/src/site-memory/commands.test.ts +++ b/src/site-memory/commands.test.ts @@ -1,8 +1,13 @@ import { Command } from 'commander'; import { describe, expect, it, vi } from 'vitest'; import { applyUnknownOptionContract, CommanderStructuralError } from '../command-surface.js'; -import { ArgumentError } from '../errors.js'; -import { readSitePutSource, registerSiteCommands, type SiteMemoryBackend } from './commands.js'; +import { ArgumentError, CliError, EXIT_CODES } from '../errors.js'; +import { + readSitePutSource, + registerSiteCommands, + type SiteLearningBackend, + type SiteMemoryBackend, +} from './commands.js'; function backend(overrides: Partial = {}): SiteMemoryBackend { return { @@ -19,13 +24,83 @@ function backend(overrides: Partial = {}): SiteMemoryBackend }; } -function program(store: SiteMemoryBackend, io?: { readStdin?: () => Promise }): Command { +const PRODUCT = { + key: 'example.test', + hostname: 'example.test', + displayHostname: 'example.test', + registrableDomain: 'example.test', +}; + +const SUMMARY = { + id: '20260831T142300Z-aaaa', + domain: 'example.test', + hostname: 'example.test', + observedAt: '2026-08-31T14:23:00.000Z', + observedDateUtc: '2026-08-31', + kind: 'access', + claim: 'Login is optional for /hot', + consequence: 'Skip auth for listing', + status: 'pending' as const, +}; + +function learning(overrides: Partial = {}): SiteLearningBackend { + return { + context: vi.fn(async () => ({ + resolution: { status: 'new' as const, requested: PRODUCT, product: PRODUCT, readOnly: false }, + revision: 'rev1', + siteMarkdown: '# Example', + references: [{ path: 'sitemap/references/alt.md' }], + draftPath: '/tmp/drafts/task-1/example.test/sitemap', + readOnly: false, + diagnostics: [], + })), + addCandidate: vi.fn(async () => SUMMARY), + searchCandidates: vi.fn(async () => [SUMMARY]), + showCandidate: vi.fn(async () => ({ + schemaVersion: 1 as const, + ...SUMMARY, + evidence: 'Opened /hot without login', + environment: { publicIp: '203.0.113.9', machine: 'secret-host' }, + evidenceRole: null, + memoryCommit: null, + reviewedAt: null, + rejectionReason: null, + })), + listCandidates: vi.fn(async () => [SUMMARY]), + checkpoint: vi.fn(async () => ({ status: 'committed' as const, memoryCommit: 'mem1', provenanceCommit: 'prov1' })), + ...overrides, + }; +} + +function program( + store: SiteMemoryBackend, + io?: { readStdin?: () => Promise }, + learn?: SiteLearningBackend, +): Command { const root = new Command('webcmd').exitOverride(); - registerSiteCommands(root, store, undefined, io); + registerSiteCommands(root, store, undefined, io, learn); applyUnknownOptionContract(root); return root; } +function leaf(root: Command, path: string[]): Command { + let command: Command | undefined = root; + for (const segment of path) command = command?.commands.find(child => child.name() === segment); + if (!command) throw new Error(`missing command: ${path.join(' ')}`); + return command; +} + +async function jsonOf(argv: string[], learn: SiteLearningBackend = learning()): Promise { + const logged: unknown[] = []; + const spy = vi.spyOn(console, 'log').mockImplementation((value: unknown) => { logged.push(value); }); + try { + await program(backend(), undefined, learn).parseAsync(argv, { from: 'user' }); + return JSON.parse(String(logged[0])); + } finally { + spy.mockRestore(); + } +} + describe('site memory format flags', () => { it('documents the site argument grammar in site-memory help', () => { const help = program(backend()).commands.find(command => command.name() === 'site')!.helpInformation(); @@ -143,3 +218,119 @@ describe('readSitePutSource', () => { await expect(readSitePutSource({})).rejects.toThrow(/--stdin/); }); }); + +describe('local learning command contract', () => { + const learn = () => learning(); + + it('registers exact positional grammar only when a learning backend is provided', () => { + expect(() => leaf(program(backend()), ['site', 'memory', 'context'])).toThrow(/missing command/); + const root = program(backend(), undefined, learn()); + expect(leaf(root, ['site', 'memory', 'context']).helpInformation()).toMatch(/Usage: .*memory context \[options\] /); + expect(leaf(root, ['site', 'memory', 'candidate', 'add']).helpInformation()).toMatch(/Usage: .*candidate add \[options\] /); + expect(leaf(root, ['site', 'memory', 'candidate', 'search']).helpInformation()).toMatch(/Usage: .*candidate search \[options\] /); + expect(leaf(root, ['site', 'memory', 'candidate', 'show']).helpInformation()).toMatch(/Usage: .*candidate show \[options\] /); + expect(leaf(root, ['site', 'memory', 'candidate', 'list']).helpInformation()).toMatch(/Usage: .*candidate list \[options\] /); + expect(leaf(root, ['site', 'memory', 'checkpoint']).helpInformation()).toMatch(/Usage: .*memory checkpoint \[options\] /); + }); + + it.each([ + { argv: ['site', 'memory', 'context', 'https://example.test/'], flag: '--task-id' }, + { argv: ['site', 'memory', 'candidate', 'add', 'example.test'], flag: '--kind' }, + { argv: ['site', 'memory', 'candidate', 'search', 'example.test'], flag: '--query' }, + { argv: ['site', 'memory', 'checkpoint', 'example.test'], flag: '--expected-revision' }, + ])('rejects $argv.1 $argv.2 without $flag', async ({ argv, flag }) => { + await expect(program(backend(), undefined, learn()).parseAsync(argv, { from: 'user' })) + .rejects.toBeInstanceOf(CommanderStructuralError); + expect(leaf(program(backend(), undefined, learn()), argv.slice(0, -1)).helpInformation()).toContain(flag); + }); + + it('rejects unknown flags with the valid set for candidate add', async () => { + try { + await program(backend(), undefined, learn()).parseAsync([ + 'site', 'memory', 'candidate', 'add', 'example.test', + '--kind', 'access', '--claim', 'c', '--evidence', 'e', '--consequence', 'q', '--nope', + ], { from: 'user' }); + expect.unreachable(); + } catch (error) { + expect(error).toBeInstanceOf(CommanderStructuralError); + const output = (error as CommanderStructuralError).output; + expect(output).toContain("unknown option '--nope'"); + expect(output).toContain('--kind'); + expect(output).toContain('--json'); + } + }); + + it('enumerates valid candidate kinds', async () => { + await expect(program(backend(), undefined, learn()).parseAsync([ + 'site', 'memory', 'candidate', 'add', 'example.test', + '--kind', 'secret', '--claim', 'c', '--evidence', 'e', '--consequence', 'q', + ], { from: 'user' })).rejects.toMatchObject({ + code: 'ARGUMENT', + message: expect.stringMatching(/action_space, better_path, access, high_consequence, repeated_mistake/), + }); + }); + + it('returns structured JSON for context, candidate inventory, and checkpoint', async () => { + const store = learn(); + expect(await jsonOf(['site', 'memory', 'context', 'https://example.test/', '--task-id', 'task-1'], store)).toEqual( + expect.objectContaining({ + revision: 'rev1', + siteMarkdown: '# Example', + references: [{ path: 'sitemap/references/alt.md' }], + draftPath: '/tmp/drafts/task-1/example.test/sitemap', + readOnly: false, + diagnostics: [], + }), + ); + expect(await jsonOf([ + 'site', 'memory', 'candidate', 'add', 'example.test', + '--kind', 'access', '--claim', 'c', '--evidence', 'e', '--consequence', 'q', + ], store)).toEqual(SUMMARY); + expect(await jsonOf(['site', 'memory', 'candidate', 'search', 'example.test', '--query', 'login'], store)).toEqual([SUMMARY]); + expect(await jsonOf(['site', 'memory', 'candidate', 'list', 'example.test'], store)).toEqual([SUMMARY]); + expect(JSON.stringify(await jsonOf(['site', 'memory', 'candidate', 'search', 'example.test', '--query', 'login'], store))) + .not.toMatch(/203\.0\.113\.9|secret-host|environment/); + const shown = await jsonOf(['site', 'memory', 'candidate', 'show', 'example.test', SUMMARY.id], store) as { environment: unknown }; + expect(shown.environment).toEqual({ publicIp: '203.0.113.9', machine: 'secret-host' }); + expect(await jsonOf([ + 'site', 'memory', 'checkpoint', 'example.test', + '--task-id', 'task-1', '--expected-revision', 'rev1', '--reason', 'direct_correction', '--paths', 'sitemap/SITE.md', + ], store)).toEqual({ status: 'committed', memoryCommit: 'mem1', provenanceCommit: 'prov1' }); + }); + + it('hides candidates from ordinary memory list and show', async () => { + const listed = await jsonOf(['site', 'memory', 'list', 'example.test', '-f', 'json']); + const shown = await jsonOf(['site', 'memory', 'show', 'example.test', '-f', 'json']); + expect(JSON.stringify({ listed, shown })).not.toMatch(/candidates\/|203\.0\.113\.9/); + expect(await jsonOf(['site', 'memory', 'candidate', 'list', 'example.test'])).toEqual([SUMMARY]); + }); + + it('returns SITE_MEMORY_CONFLICT with revision details', async () => { + const store = learning({ + checkpoint: vi.fn(async () => ({ status: 'conflict' as const, expectedRevision: 'old', actualRevision: 'new' })), + }); + await expect(program(backend(), undefined, store).parseAsync([ + 'site', 'memory', 'checkpoint', 'example.test', + '--task-id', 'task-1', '--expected-revision', 'old', '--reason', 'direct_correction', '--paths', 'sitemap/SITE.md', + ], { from: 'user' })).rejects.toMatchObject({ + code: 'SITE_MEMORY_CONFLICT', + exitCode: EXIT_CODES.TEMPFAIL, + details: { expectedRevision: 'old', actualRevision: 'new' }, + }); + }); + + it('returns SITE_MEMORY_NOT_FOUND for a missing candidate show', async () => { + const store = learning({ + showCandidate: vi.fn(async () => { throw new Error('Candidate missing was not found.'); }), + }); + await expect(program(backend(), undefined, store).parseAsync([ + 'site', 'memory', 'candidate', 'show', 'example.test', 'missing', + ], { from: 'user' })).rejects.toBeInstanceOf(CliError); + await expect(program(backend(), undefined, store).parseAsync([ + 'site', 'memory', 'candidate', 'show', 'example.test', 'missing', + ], { from: 'user' })).rejects.toMatchObject({ + code: 'SITE_MEMORY_NOT_FOUND', + exitCode: EXIT_CODES.EMPTY_RESULT, + }); + }); +}); diff --git a/src/site-memory/commands.ts b/src/site-memory/commands.ts index b69110d0..9d261c53 100644 --- a/src/site-memory/commands.ts +++ b/src/site-memory/commands.ts @@ -5,6 +5,9 @@ import { ArgumentError, CliError, EXIT_CODES } from '../errors.js'; import { getRequestedHelpFormat } from '../help.js'; import { render as renderOutput } from '../output.js'; import { writeToStream } from '../stream-write.js'; +import { addCandidate, listCandidates, searchCandidates, showCandidate } from './candidates.js'; +import { checkpointMemory } from './checkpoint.js'; +import { getMemoryContext } from './context.js'; import { addFieldMapping, addResponseSample, @@ -19,6 +22,15 @@ import { type SiteMemoryBody, type SiteMemoryListing, } from './local-store.js'; +import { + CANDIDATE_KINDS, + type Candidate, + type CandidateDisposition, + type CandidateSummary, + type CheckpointReason, + type CheckpointResult, + type MemoryContext, +} from './model.js'; type JsonObject = Record; type MemoryKind = 'notes' | 'endpoints' | 'field-map' | 'verify' | 'fixture'; @@ -35,6 +47,30 @@ export interface SiteMemoryBackend { sample(site: string, command: string, body: string): Promise; } +export interface SiteLearningBackend { + context(url: string, taskId: string): Promise; + addCandidate(input: { + product: string; + hostname?: string; + kind: string; + claim: string; + evidence: string; + consequence: string; + observedAt?: string; + }): Promise; + searchCandidates(product: string, query: string, limit?: number): Promise; + showCandidate(product: string, id: string): Promise; + listCandidates(product: string): Promise; + checkpoint(input: { + product: string; + taskId: string; + expectedRevision: string | null; + reason: CheckpointReason; + paths: string[]; + dispositions?: CandidateDisposition[]; + }): Promise; +} + export interface SiteCommandIo { readStdin?(): Promise; } @@ -74,6 +110,7 @@ export function registerSiteCommands( backend: SiteMemoryBackend, stdout?: NodeJS.WritableStream, io: SiteCommandIo = {}, + learning?: SiteLearningBackend, ): void { const site = withHelpFooter(root.command('site') .description('Read and write per-site memory: notes, verified endpoints, field maps, fixtures and samples') @@ -107,6 +144,7 @@ export function registerSiteCommands( list.action(async (name, opts: { output?: string; format?: string }) => { await emitListing(list, await backend.list(name), opts, stdout, ['path', 'updatedAt', 'byteSize', 'sha256']); }); + if (learning) registerLearningCommands(memory, learning, stdout); /** Write commands print nothing by default; a format flag turns that into a result object. */ const emitWriteResult = async (command: Command, payload: Record): Promise => { @@ -260,6 +298,132 @@ export function registerSiteCommands( }); } +function registerLearningCommands( + memory: Command, + learning: SiteLearningBackend, + stdout?: NodeJS.WritableStream, +): void { + const emit = async (command: Command, data: unknown, opts: { format?: string } = {}): Promise => { + const fmt = resolveCommandOutputFormat(command, opts.format); + if (fmt === null) return; + await renderOutput(data, { fmt, fmtExplicit: true, stdout }); + }; + + const context = addOutputFormatOption(withExample(memory.command('context') + .description('Resolve product identity, seed memory once, and return the task draft path') + .argument('', 'Page URL used to resolve the product') + .requiredOption('--task-id ', 'Task id that owns the isolated draft'), + 'webcmd site memory context https://example.test/ --task-id task-1 -f json'), 'json'); + context.action(async (url: string, opts: { taskId: string; format?: string }) => { + await emit(context, await learning.context(url, opts.taskId), opts); + }); + + const candidate = withExample(memory.command('candidate') + .description('Capture and inspect candidate evidence: webcmd site memory candidate ') + .usage('add|search|show|list [args] [options]'), + 'webcmd site memory candidate list example.test -f json'); + + const add = addOutputFormatOption(withExample(candidate.command('add') + .description('Record one qualifying observation as candidate evidence') + .argument('', 'Product key or hostname') + .requiredOption('--kind ', `One of: ${CANDIDATE_KINDS.join(', ')}`) + .requiredOption('--claim ', 'Short claim this observation supports') + .requiredOption('--evidence ', 'Bounded secret-free evidence from the task') + .requiredOption('--consequence ', 'Why this may matter later') + .option('--hostname ', 'Observed hostname when it differs from the product key') + .option('--observed-at ', 'Observation timestamp; defaults to now'), + 'webcmd site memory candidate add example.test --kind access --claim "Login is optional" --evidence "Opened /hot" --consequence "Skip auth" -f json'), 'json'); + add.action(async (product: string, opts: { + kind: string; claim: string; evidence: string; consequence: string; hostname?: string; observedAt?: string; format?: string; + }) => { + await emit(add, await learning.addCandidate({ + product, + kind: parseCandidateKind(opts.kind), + claim: opts.claim, + evidence: opts.evidence, + consequence: opts.consequence, + ...(opts.hostname ? { hostname: opts.hostname } : {}), + ...(opts.observedAt ? { observedAt: opts.observedAt } : {}), + }), opts); + }); + + const search = addOutputFormatOption(withExample(candidate.command('search') + .description('Search pending candidates with bounded lexical matching') + .argument('', 'Product key or hostname') + .requiredOption('--query ', 'Lexical query over claim, kind, hostname, and consequence') + .option('--limit ', 'Maximum matches to return'), + 'webcmd site memory candidate search example.test --query "old reddit" -f json'), 'json'); + search.action(async (product: string, opts: { query: string; limit?: string; format?: string }) => { + await emit(search, await learning.searchCandidates(product, opts.query, parseLimit(opts.limit)), opts); + }); + + const show = addOutputFormatOption(withExample(candidate.command('show') + .description('Load one explicit candidate, including environment provenance') + .argument('', 'Product key or hostname') + .argument('', 'Candidate id'), + 'webcmd site memory candidate show example.test 20260831T142300Z-aaaa -f json'), 'json'); + show.action(async (product: string, id: string, opts: { format?: string }) => { + try { + await emit(show, await learning.showCandidate(product, id), opts); + } catch (error) { + throw notFoundOrRethrow(error); + } + }); + + const candidateList = addOutputFormatOption(withExample(candidate.command('list') + .description('List candidate inventory for a product without raw environment values') + .argument('', 'Product key or hostname'), + 'webcmd site memory candidate list example.test -f json'), 'json'); + candidateList.action(async (product: string, opts: { format?: string }) => { + await emit(candidateList, await learning.listCandidates(product), opts); + }); + + const checkpoint = addOutputFormatOption(withExample(memory.command('checkpoint') + .description('Publish a task draft into active memory with explicit candidate dispositions') + .argument('', 'Product key or hostname') + .requiredOption('--task-id ', 'Task id whose draft should be published') + .requiredOption('--expected-revision ', 'Revision returned by site memory context; use null when none') + .requiredOption('--reason ', 'candidate_ingestion, direct_correction, or major_rewrite') + .requiredOption('--paths ', 'Comma-separated Markdown paths to copy from the draft') + .option('--dispositions ', 'JSON array of candidate dispositions'), + 'webcmd site memory checkpoint example.test --task-id task-1 --expected-revision rev1 --reason direct_correction --paths sitemap/SITE.md -f json'), 'json'); + checkpoint.action(async (product: string, opts: { + taskId: string; expectedRevision: string; reason: string; paths: string; dispositions?: string; format?: string; + }) => { + const result = await learning.checkpoint({ + product, + taskId: opts.taskId, + expectedRevision: parseRevision(opts.expectedRevision), + reason: parseCheckpointReason(opts.reason), + paths: parsePaths(opts.paths), + ...(opts.dispositions ? { dispositions: parseDispositions(opts.dispositions) } : {}), + }); + if (result.status === 'conflict') { + throw Object.assign( + new CliError( + 'SITE_MEMORY_CONFLICT', + 'Expected revision changed.', + 'Retry webcmd site memory context, then checkpoint once.', + EXIT_CODES.TEMPFAIL, + ), + { details: { expectedRevision: result.expectedRevision, actualRevision: result.actualRevision } }, + ); + } + await emit(checkpoint, result, opts); + }); +} + +export function createLocalLearningBackend(options: LocalStoreOptions = {}): SiteLearningBackend { + return { + context: (url, taskId) => getMemoryContext({ url, taskId, ...options }), + addCandidate: input => addCandidate({ ...input, ...options }), + searchCandidates: (product, query, limit) => searchCandidates(product, query, limit, options), + showCandidate: (product, id) => showCandidate(product, id, options), + listCandidates: product => listCandidates(product, options), + checkpoint: input => checkpointMemory({ ...input, ...options }), + }; +} + export function createLocalSiteMemoryBackend(options: LocalStoreOptions = {}): SiteMemoryBackend { return { show: async (site, kind) => (await showSiteMemory(site, options)).filter(item => !kind || kindForPath(item.path) === kind), @@ -302,6 +466,48 @@ function parseKind(value: string | undefined): MemoryKind | undefined { throw new ArgumentError('--kind must be notes, endpoints, field-map, verify, or fixture.'); } +function parseCandidateKind(value: string): (typeof CANDIDATE_KINDS)[number] { + if ((CANDIDATE_KINDS as readonly string[]).includes(value)) return value as (typeof CANDIDATE_KINDS)[number]; + throw new ArgumentError(`--kind must be one of: ${CANDIDATE_KINDS.join(', ')}.`); +} + +const CHECKPOINT_REASONS = ['candidate_ingestion', 'direct_correction', 'major_rewrite'] as const; + +function parseCheckpointReason(value: string): CheckpointReason { + if ((CHECKPOINT_REASONS as readonly string[]).includes(value)) return value as CheckpointReason; + throw new ArgumentError(`--reason must be one of: ${CHECKPOINT_REASONS.join(', ')}.`); +} + +function parseRevision(value: string): string | null { + return value === '' || value === 'null' ? null : value; +} + +function parsePaths(value: string): string[] { + return value.split(',').map(path => path.trim()).filter(Boolean); +} + +function parseDispositions(value: string): CandidateDisposition[] { + try { + const parsed = JSON.parse(value) as unknown; + if (Array.isArray(parsed)) return parsed as CandidateDisposition[]; + } catch { /* covered by the shared message below */ } + throw new ArgumentError('--dispositions must be a JSON array.'); +} + +function parseLimit(value: string | undefined): number | undefined { + if (value === undefined) return undefined; + const parsed = Number.parseInt(value, 10); + if (!Number.isSafeInteger(parsed) || parsed < 1) throw new ArgumentError('--limit must be a positive integer.'); + return parsed; +} + +function notFoundOrRethrow(error: unknown): never { + if (error instanceof Error && /not found/i.test(error.message)) { + throw new CliError('SITE_MEMORY_NOT_FOUND', error.message, undefined, EXIT_CODES.EMPTY_RESULT); + } + throw error; +} + function parseJsonObject(value: string): JsonObject { try { const parsed = JSON.parse(value) as unknown; From 054a450f498d640fb2e8215347b635c9c0f97f07 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Tue, 1 Sep 2026 01:25:37 +0530 Subject: [PATCH 16/41] fix(site-memory): validate learning command inputs --- src/site-memory/commands.test.ts | 76 ++++++++++++++++++++++++++++++++ src/site-memory/commands.ts | 66 ++++++++++++++++++++++++--- 2 files changed, 136 insertions(+), 6 deletions(-) diff --git a/src/site-memory/commands.test.ts b/src/site-memory/commands.test.ts index deda2ab3..afa4e5aa 100644 --- a/src/site-memory/commands.test.ts +++ b/src/site-memory/commands.test.ts @@ -334,3 +334,79 @@ describe('local learning command contract', () => { }); }); }); + +describe('learning command trust-boundary parsers', () => { + const checkpoint = (...extra: string[]) => [ + 'site', 'memory', 'checkpoint', 'example.test', + '--task-id', 'task-1', '--expected-revision', 'rev1', '--reason', 'direct_correction', + ...extra, + ]; + + async function expectArgument(argv: string[]): Promise { + await expect(program(backend(), undefined, learning()).parseAsync(argv, { from: 'user' })) + .rejects.toMatchObject({ code: 'ARGUMENT' }); + } + + it.each(['1x', '1.5', '+1', '-1', '0', String(Number.MAX_SAFE_INTEGER + 1), '9'.repeat(400)])( + 'rejects --limit %s', + async (limit) => { + await expectArgument(['site', 'memory', 'candidate', 'search', 'example.test', '--query', 'login', '--limit', limit]); + }, + ); + + it('forwards a whole positive --limit', async () => { + const store = learning(); + await program(backend(), undefined, store).parseAsync( + ['site', 'memory', 'candidate', 'search', 'example.test', '--query', 'login', '--limit', '3'], + { from: 'user' }, + ); + expect(store.searchCandidates).toHaveBeenCalledWith('example.test', 'login', 3); + }); + + it.each(['', ',', 'sitemap/SITE.md,', ',sitemap/SITE.md', 'sitemap/SITE.md,,other.md', 'a, a', 'a,b,a'])( + 'rejects --paths %s', + async (paths) => { + await expectArgument(checkpoint('--paths', paths)); + }, + ); + + it('forwards unique nonempty --paths', async () => { + const store = learning(); + await program(backend(), undefined, store).parseAsync( + checkpoint('--paths', 'sitemap/SITE.md, sitemap/other.md'), + { from: 'user' }, + ); + expect(store.checkpoint).toHaveBeenCalledWith(expect.objectContaining({ + paths: ['sitemap/SITE.md', 'sitemap/other.md'], + })); + }); + + it.each([ + 'not-json', + '{}', + '[1]', + '[[]]', + '[{"status":"rejected"}]', + '[{"id":"","status":"rejected"}]', + '[{"id":"cand-1","status":"pending"}]', + '[{"id":"cand-1","status":"ingested","evidenceRole":"maybe"}]', + '[{"id":"cand-1","status":"rejected","conflictsWithMemory":"yes"}]', + '[{"id":"cand-1","status":"rejected","extra":true}]', + '[{"id":"cand-1","status":"rejected","rejectionReason":"password := hunter2"}]', + ])('rejects --dispositions %s', async (dispositions) => { + await expectArgument(checkpoint('--paths', 'sitemap/SITE.md', '--dispositions', dispositions)); + }); + + it('forwards typed --dispositions without checkpoint field coupling', async () => { + const store = learning(); + const dispositions = [ + { id: 'cand-1', status: 'ingested' as const, evidenceRole: 'supporting' as const, conflictsWithMemory: false }, + { id: 'cand-2', status: 'rejected' as const, rejectionReason: 'stale', evidenceRole: null }, + ]; + await program(backend(), undefined, store).parseAsync( + checkpoint('--paths', 'sitemap/SITE.md', '--dispositions', JSON.stringify(dispositions)), + { from: 'user' }, + ); + expect(store.checkpoint).toHaveBeenCalledWith(expect.objectContaining({ dispositions })); + }); +}); diff --git a/src/site-memory/commands.ts b/src/site-memory/commands.ts index 9d261c53..d3d86cff 100644 --- a/src/site-memory/commands.ts +++ b/src/site-memory/commands.ts @@ -483,20 +483,74 @@ function parseRevision(value: string): string | null { } function parsePaths(value: string): string[] { - return value.split(',').map(path => path.trim()).filter(Boolean); + const paths = value.split(',').map(path => path.trim()); + if (paths.some(path => path === '')) throw new ArgumentError('--paths requires nonempty explicit paths.'); + if (new Set(paths).size !== paths.length) throw new ArgumentError('--paths must not contain duplicates.'); + return paths; } +const DISPOSITION_KEYS = new Set(['id', 'status', 'evidenceRole', 'rejectionReason', 'conflictsWithMemory']); +const SECRET_KEY = /^(password|passwd|secret|token|cookie|cookies|authorization|api[_-]?key|set-cookie)$/i; +const SECRET_TEXT = /(password\s*[:=]|secret\s*[:=]|api[_-]?key|authorization\s*:|bearer\s+\S+|cookie\s*[:=])/i; + function parseDispositions(value: string): CandidateDisposition[] { + let parsed: unknown; try { - const parsed = JSON.parse(value) as unknown; - if (Array.isArray(parsed)) return parsed as CandidateDisposition[]; - } catch { /* covered by the shared message below */ } - throw new ArgumentError('--dispositions must be a JSON array.'); + parsed = JSON.parse(value); + } catch { + throw new ArgumentError('--dispositions must be a JSON array.'); + } + if (!Array.isArray(parsed)) throw new ArgumentError('--dispositions must be a JSON array.'); + return parsed.map(parseDisposition); +} + +function isPlainObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function parseDisposition(value: unknown): CandidateDisposition { + if (!isPlainObject(value)) throw new ArgumentError('--dispositions must be a JSON array of objects.'); + for (const key of Object.keys(value)) { + if (SECRET_KEY.test(key) || !DISPOSITION_KEYS.has(key)) { + throw new ArgumentError('--dispositions contains an unknown or secret-bearing field.'); + } + } + const id = value.id; + if (typeof id !== 'string' || id.trim() === '' || SECRET_TEXT.test(id)) { + throw new ArgumentError('--dispositions id must be a nonempty secret-free string.'); + } + const status = value.status; + if (status !== 'ingested' && status !== 'rejected') { + throw new ArgumentError('--dispositions status must be ingested or rejected.'); + } + const row: CandidateDisposition = { id, status }; + if ('evidenceRole' in value) { + const role = value.evidenceRole; + if (role !== null && role !== 'supporting' && role !== 'dissenting') { + throw new ArgumentError('--dispositions evidenceRole must be supporting, dissenting, or null.'); + } + row.evidenceRole = role; + } + if ('rejectionReason' in value) { + const reason = value.rejectionReason; + if (reason !== null && (typeof reason !== 'string' || SECRET_TEXT.test(reason))) { + throw new ArgumentError('--dispositions rejectionReason must be a secret-free string or null.'); + } + row.rejectionReason = reason; + } + if ('conflictsWithMemory' in value) { + if (typeof value.conflictsWithMemory !== 'boolean') { + throw new ArgumentError('--dispositions conflictsWithMemory must be a boolean.'); + } + row.conflictsWithMemory = value.conflictsWithMemory; + } + return row; } function parseLimit(value: string | undefined): number | undefined { if (value === undefined) return undefined; - const parsed = Number.parseInt(value, 10); + if (!/^\d+$/.test(value)) throw new ArgumentError('--limit must be a positive integer.'); + const parsed = Number(value); if (!Number.isSafeInteger(parsed) || parsed < 1) throw new ArgumentError('--limit must be a positive integer.'); return parsed; } From 870416ca56be012fd26baeb969d359406815bc8d Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Tue, 1 Sep 2026 01:27:27 +0530 Subject: [PATCH 17/41] fix(site-memory): remove obsolete sitemap resolver --- src/cli.test.ts | 60 +------------------------------------------------ src/cli.ts | 39 -------------------------------- 2 files changed, 1 insertion(+), 98 deletions(-) diff --git a/src/cli.test.ts b/src/cli.test.ts index 4000b1b8..b246df20 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -70,7 +70,7 @@ vi.mock('node:child_process', async () => { }); import { handleProgramParseError } from './cli-error-report.js'; -import { createProgram, findPackageRoot, loadAntigravityServe, normalizeVerifyRows, renderVerifyPreview, resolveBrowserVerifyInvocation, resolveSitemapAvailabilityForUrl, selectFreshByTimestamp } from './cli.js'; +import { createProgram, findPackageRoot, loadAntigravityServe, normalizeVerifyRows, renderVerifyPreview, resolveBrowserVerifyInvocation, selectFreshByTimestamp } from './cli.js'; const realHome = process.env.HOME; const realConfigDir = process.env.WEBCMD_CONFIG_DIR; @@ -1549,64 +1549,6 @@ describe('selectFreshByTimestamp', () => { }); }); -describe('resolveSitemapAvailabilityForUrl', () => { - it('resolves the product key and local SITE.md without registry names', () => { - const homeDir = path.join(os.tmpdir(), 'webcmd-sitemap-home'); - const localSitemap = path.join(homeDir, '.webcmd', 'sites', 'news.ycombinator.com', 'sitemap', 'SITE.md'); - - const report = resolveSitemapAvailabilityForUrl('https://news.ycombinator.com/item?id=1', { - homeDir, - fileExists: (candidate) => candidate === localSitemap, - }); - - expect(report).toMatchObject({ - site: 'news.ycombinator.com', - available: true, - source: 'local', - paths: { local: localSitemap }, - }); - expect(report?.hint).toContain('site memory context'); - expect(JSON.stringify(report)).not.toMatch(/local\+global|hackernews|webcmd-browser-sitemap/); - }); - - it('ignores package sitemaps and registry aliases', () => { - const homeDir = path.join(os.tmpdir(), 'webcmd-sitemap-home'); - const packageRoot = path.join(os.tmpdir(), 'webcmd-sitemap-package'); - const packageSitemap = path.join(packageRoot, 'sitemaps', 'twitter'); - const aliasSitemap = path.join(homeDir, '.webcmd', 'sites', 'twitter', 'sitemap.md'); - - const report = resolveSitemapAvailabilityForUrl('https://x.com/webcmd', { - homeDir, - fileExists: (candidate) => candidate === packageSitemap || candidate === aliasSitemap, - }); - - expect(report).toBeNull(); - }); - - it('returns availability for existing SITE.md even when learning is read-only', () => { - const homeDir = path.join(os.tmpdir(), 'webcmd-sitemap-home'); - const localSitemap = path.join(homeDir, '.webcmd', 'sites', 'example.test', 'sitemap', 'SITE.md'); - - expect(() => resolveSitemapAvailabilityForUrl('https://example.test/', { - homeDir, - fileExists: (candidate) => candidate === localSitemap, - })).not.toThrow(); - expect(resolveSitemapAvailabilityForUrl('https://example.test/', { - homeDir, - fileExists: (candidate) => candidate === localSitemap, - })?.available).toBe(true); - }); - - it('returns null when no local SITE.md exists', () => { - const report = resolveSitemapAvailabilityForUrl('https://example.com/', { - homeDir: path.join(os.tmpdir(), 'webcmd-sitemap-home'), - fileExists: () => false, - }); - - expect(report).toBeNull(); - }); -}); - describe('local learning command registration', () => { it('registers site memory context, candidate, and checkpoint on the local program', () => { const program = createProgram('', ''); diff --git a/src/cli.ts b/src/cli.ts index a1a8fcf8..7ee521c3 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -64,7 +64,6 @@ import { classifyCommandOrigin, formatCommandOrigin } from './command-origin.js' import { readOverrideRecords, removeOverrideRecords } from './override-provenance.js'; import { clearDaemonRunContext, generateRunId, isUnknownOutcomeError, runWithDaemonRunContext } from './session-lease.js'; import { createLocalLearningBackend, createLocalSiteMemoryBackend, registerSiteCommands } from './site-memory/commands.js'; -import { canonicalProductKey } from './site-memory/product-resolver.js'; import { resolveAdapterSourcePath, splitAdapterCommandKey } from './adapter-source.js'; const CLI_FILE = fileURLToPath(import.meta.url); @@ -323,44 +322,6 @@ export type SiteMemoryReport = { notes: { present: boolean; path: string }; }; -export type SitemapAvailability = { - site: string; - available: true; - source: 'local'; - hint: string; - paths: { - local?: string; - }; -}; - -type SitemapAvailabilityOptions = { - homeDir?: string; - fileExists?: (candidate: string) => boolean; -}; - -const SITEMAP_HINT = - 'Product sitemap available. Use `webcmd site memory context -f json` for navigation context; treat browser state as truth if it disagrees.'; - -export function resolveSitemapAvailabilityForUrl(url: string, options: SitemapAvailabilityOptions = {}): SitemapAvailability | null { - let product; - try { - product = canonicalProductKey(url); - } catch { - return null; - } - const homeDir = options.homeDir ?? os.homedir(); - const fileExists = options.fileExists ?? fs.existsSync; - const local = path.join(homeDir, '.webcmd', 'sites', product.key, 'sitemap', 'SITE.md'); - if (!fileExists(local)) return null; - return { - site: product.key, - available: true, - source: 'local', - hint: SITEMAP_HINT, - paths: { local }, - }; -} - export function checkSiteMemory(site: string): SiteMemoryReport { const siteDir = path.join(os.homedir(), '.webcmd', 'sites', site); const endpointsPath = path.join(siteDir, 'endpoints.json'); From 0ba83367662a04b0f902101acc1facbb2972814c Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Tue, 1 Sep 2026 01:36:31 +0530 Subject: [PATCH 18/41] feat(skills): ship one self-learning browser skill --- mcp-skills/smart-search.md | 40 -- mcp-skills/webcmd-adapter-author.md | 61 -- mcp-skills/webcmd-autofix.md | 48 -- mcp-skills/webcmd-browser-sitemap.md | 39 -- mcp-skills/webcmd-browser.md | 25 +- mcp-skills/webcmd-sitemap-author.md | 56 -- mcp-skills/webcmd-usage.md | 132 ----- skill-src/cli/smart-search/SKILL.src.md | 168 ------ .../cli/webcmd-adapter-author/SKILL.src.md | 284 ---------- .../references/adapter-template.src.md | 302 ---------- .../references/api-discovery.src.md | 255 --------- .../references/coverage-matrix.src.md | 47 -- .../references/field-conventions.src.md | 89 --- .../references/field-decode-playbook.src.md | 110 ---- .../references/jsdom-fixture-pattern.src.md | 126 ----- .../references/output-design.src.md | 71 --- .../references/site-memory.src.md | 167 ------ .../references/site-recon.src.md | 175 ------ .../references/strategy-selection.src.md | 121 ---- .../references/success-rate-pitfalls.src.md | 137 ----- .../references/typed-errors.src.md | 143 ----- skill-src/cli/webcmd-autofix/SKILL.src.md | 316 ----------- .../cli/webcmd-browser-sitemap/SKILL.src.md | 105 ---- skill-src/cli/webcmd-browser/SKILL.src.md | 50 +- .../references/browser-run-playwright.src.md | 2 + .../references/candidate-schema.src.md | 28 + .../references/git-lifecycle.src.md | 25 + .../references/sitemap-memory.src.md | 18 + .../cli/webcmd-sitemap-author/SKILL.src.md | 161 ------ .../references/sitemap-schema.src.md | 520 ------------------ skill-src/cli/webcmd-usage/SKILL.src.md | 304 ---------- skill-src/mcp/smart-search.src.md | 40 -- skill-src/mcp/webcmd-adapter-author.src.md | 58 -- skill-src/mcp/webcmd-autofix.src.md | 48 -- skill-src/mcp/webcmd-browser-sitemap.src.md | 39 -- skill-src/mcp/webcmd-browser.src.md | 25 +- skill-src/mcp/webcmd-sitemap-author.src.md | 56 -- skill-src/mcp/webcmd-usage.src.md | 132 ----- skills/smart-search/SKILL.md | 152 ----- skills/webcmd-adapter-author/SKILL.md | 278 ---------- .../references/adapter-template.md | 302 ---------- .../references/api-discovery.md | 255 --------- .../references/coverage-matrix.md | 47 -- .../references/field-conventions.md | 89 --- .../references/field-decode-playbook.md | 110 ---- .../references/jsdom-fixture-pattern.md | 126 ----- .../references/output-design.md | 71 --- .../references/site-memory.md | 167 ------ .../references/site-recon.md | 175 ------ .../references/strategy-selection.md | 121 ---- .../references/success-rate-pitfalls.md | 137 ----- .../references/typed-errors.md | 143 ----- skills/webcmd-autofix/SKILL.md | 307 ----------- skills/webcmd-browser-sitemap/SKILL.md | 96 ---- skills/webcmd-browser/SKILL.md | 46 +- .../references/browser-run-playwright.md | 2 + .../references/candidate-schema.md | 28 + .../references/git-lifecycle.md | 25 + .../references/sitemap-memory.md | 18 + skills/webcmd-sitemap-author/SKILL.md | 152 ----- .../references/sitemap-schema.md | 520 ------------------ skills/webcmd-usage/SKILL.md | 287 ---------- src/mcp-skills.test.ts | 59 +- src/skills.test.ts | 340 +++++------- 64 files changed, 342 insertions(+), 8234 deletions(-) delete mode 100644 mcp-skills/smart-search.md delete mode 100644 mcp-skills/webcmd-adapter-author.md delete mode 100644 mcp-skills/webcmd-autofix.md delete mode 100644 mcp-skills/webcmd-browser-sitemap.md delete mode 100644 mcp-skills/webcmd-sitemap-author.md delete mode 100644 mcp-skills/webcmd-usage.md delete mode 100644 skill-src/cli/smart-search/SKILL.src.md delete mode 100644 skill-src/cli/webcmd-adapter-author/SKILL.src.md delete mode 100644 skill-src/cli/webcmd-adapter-author/references/adapter-template.src.md delete mode 100644 skill-src/cli/webcmd-adapter-author/references/api-discovery.src.md delete mode 100644 skill-src/cli/webcmd-adapter-author/references/coverage-matrix.src.md delete mode 100644 skill-src/cli/webcmd-adapter-author/references/field-conventions.src.md delete mode 100644 skill-src/cli/webcmd-adapter-author/references/field-decode-playbook.src.md delete mode 100644 skill-src/cli/webcmd-adapter-author/references/jsdom-fixture-pattern.src.md delete mode 100644 skill-src/cli/webcmd-adapter-author/references/output-design.src.md delete mode 100644 skill-src/cli/webcmd-adapter-author/references/site-memory.src.md delete mode 100644 skill-src/cli/webcmd-adapter-author/references/site-recon.src.md delete mode 100644 skill-src/cli/webcmd-adapter-author/references/strategy-selection.src.md delete mode 100644 skill-src/cli/webcmd-adapter-author/references/success-rate-pitfalls.src.md delete mode 100644 skill-src/cli/webcmd-adapter-author/references/typed-errors.src.md delete mode 100644 skill-src/cli/webcmd-autofix/SKILL.src.md delete mode 100644 skill-src/cli/webcmd-browser-sitemap/SKILL.src.md create mode 100644 skill-src/cli/webcmd-browser/references/candidate-schema.src.md create mode 100644 skill-src/cli/webcmd-browser/references/git-lifecycle.src.md create mode 100644 skill-src/cli/webcmd-browser/references/sitemap-memory.src.md delete mode 100644 skill-src/cli/webcmd-sitemap-author/SKILL.src.md delete mode 100644 skill-src/cli/webcmd-sitemap-author/references/sitemap-schema.src.md delete mode 100644 skill-src/cli/webcmd-usage/SKILL.src.md delete mode 100644 skill-src/mcp/smart-search.src.md delete mode 100644 skill-src/mcp/webcmd-adapter-author.src.md delete mode 100644 skill-src/mcp/webcmd-autofix.src.md delete mode 100644 skill-src/mcp/webcmd-browser-sitemap.src.md delete mode 100644 skill-src/mcp/webcmd-sitemap-author.src.md delete mode 100644 skill-src/mcp/webcmd-usage.src.md delete mode 100644 skills/smart-search/SKILL.md delete mode 100644 skills/webcmd-adapter-author/SKILL.md delete mode 100644 skills/webcmd-adapter-author/references/adapter-template.md delete mode 100644 skills/webcmd-adapter-author/references/api-discovery.md delete mode 100644 skills/webcmd-adapter-author/references/coverage-matrix.md delete mode 100644 skills/webcmd-adapter-author/references/field-conventions.md delete mode 100644 skills/webcmd-adapter-author/references/field-decode-playbook.md delete mode 100644 skills/webcmd-adapter-author/references/jsdom-fixture-pattern.md delete mode 100644 skills/webcmd-adapter-author/references/output-design.md delete mode 100644 skills/webcmd-adapter-author/references/site-memory.md delete mode 100644 skills/webcmd-adapter-author/references/site-recon.md delete mode 100644 skills/webcmd-adapter-author/references/strategy-selection.md delete mode 100644 skills/webcmd-adapter-author/references/success-rate-pitfalls.md delete mode 100644 skills/webcmd-adapter-author/references/typed-errors.md delete mode 100644 skills/webcmd-autofix/SKILL.md delete mode 100644 skills/webcmd-browser-sitemap/SKILL.md create mode 100644 skills/webcmd-browser/references/candidate-schema.md create mode 100644 skills/webcmd-browser/references/git-lifecycle.md create mode 100644 skills/webcmd-browser/references/sitemap-memory.md delete mode 100644 skills/webcmd-sitemap-author/SKILL.md delete mode 100644 skills/webcmd-sitemap-author/references/sitemap-schema.md delete mode 100644 skills/webcmd-usage/SKILL.md diff --git a/mcp-skills/smart-search.md b/mcp-skills/smart-search.md deleted file mode 100644 index 5132480c..00000000 --- a/mcp-skills/smart-search.md +++ /dev/null @@ -1,40 +0,0 @@ -# Smart Search through MCP - -Use this for research, source discovery, direct URL fetches, and evidence. Every -operation goes through `webcmd_cli_run` with argv data; request JSON whenever the -result will be read programmatically. - -Use live fetch results, command metadata, and command help. Do not infer command arguments from this skill, maintain a routing table, or claim a source was searched when it was not. - -## Cost order - -For a supplied URL, try the first-choice fetch path before browser work: - - { "argv": ["web", "fetch", "--url", "https://example.com", "-f", "json"] } - -Only `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER` permits browser fallback. For a -topic without a named site, fetch one search-engine result page, extract result -URLs from its JSON/text response, then fetch up to three target pages. Search -snippets discover sources; fetched primary content is evidence. - -For a named site, inspect the live command surface first: - - { "argv": ["list", "--tag", "search", "-f", "json"] } - { "argv": ["github", "search", "--help"] } - { "argv": ["github", "search", "--query", "agents", "-f", "json"] } - -Do not create shell pipelines. Read the JSON response directly, preserve source -URLs, and report failures rather than claiming an unperformed search. - -Any truncation warning means adapter discovery is incomplete: narrow the filter and inspect again. Absence from truncated output never proves that no adapter exists. - -## Budgets - -Try one search engine by default and a second only if the first is weak or -blocked. Fetch three result URLs by default (five for a broad comparison), use -at most two browser sessions, and run one adapter search unless the first is -weak or needs independent corroboration. A rate limit, login gate, CAPTCHA, or -unusable extraction is a reason to move to another source, not to repeat the -same request. - -Report the commands run, sources fetched, browser fallback URLs, and gaps. diff --git a/mcp-skills/webcmd-adapter-author.md b/mcp-skills/webcmd-adapter-author.md deleted file mode 100644 index df53d650..00000000 --- a/mcp-skills/webcmd-adapter-author.md +++ /dev/null @@ -1,61 +0,0 @@ -# Webcmd Adapter Authoring through MCP - -Author a deterministic adapter only after reconnaissance proves a reusable -workflow. Use `webcmd_cli_run` for every interaction; source is tenant-owned -virtual content, never a local checkout. - -- Adapters import only `@agentrhq/webcmd/registry` and `@agentrhq/webcmd/errors`; do not add third-party dependencies. -- Browser-run’s Playwright-style `page` and adapter `func(page,args)` are different contracts. Preserve evidence and behavior, not syntax. Implement adapters with the existing `IPage`, pipeline, Node-fetch, or interceptor APIs. -- The `columns` array and `func` return object keys must match exactly, including order. -- **Intermediate parsing object keys must not overlap any `columns` entry.** Otherwise silent-column-drop audits can misread the adapter. Use dedicated internal names and destructure with aliases when pushing rows. -Use live fetch results, command metadata, and command help. Do not infer command arguments from this skill, maintain a routing table, or claim a source was searched when it was not. - -## Reconnaissance and strategy - -Start with live capability and command help, then inspect site memory and a -bounded browser session. Prefer public or documented APIs, then stable UI/DOM -semantics; use internal page requests or interception only when the page proves -they are necessary. Record the observed request/state, authentication source, -replay result, and why a simpler strategy cannot work. - -Create a named Session and keep its immutable, Profile-scoped readable ID for -raw browser evidence: - - { "argv": ["list", "-f", "json"] } - { "argv": ["site", "memory", "show", "example", "-f", "json"] } - { "argv": ["--profile", "work", "session", "create", "Work Project", "-f", "json"] } - { "argv": ["--profile", "work", "--session", "work-project-k7", "browser", "snapshot", "--snapshot-mode", "tree", "-f", "json"] } - -Do not bypass authentication, CAPTCHA, rate limits, or access controls. An -`action_required` response belongs to the user; provide its view URL and run its -returned verifier after the user completes it. - -## Virtual adapter source - -Get the scaffold or existing source as an artifact-backed virtual file: - - { "argv": ["adapter", "source", "get", "example/search", "--output", "adapter.ts"] } - -The command materializes source at the virtual relative path `adapter.ts`. Read -that virtual file, edit it in the tool call, then put it back using the same -path and a virtual file attachment: - - { - "argv": ["adapter", "source", "put", "example/search", "adapter.ts"], - "files": [{ "path": "adapter.ts", "artifactUri": "webcmd://artifacts/exec_.../ea_..." }] - } - -The scaffold, traces, fixtures, and verification output are artifacts. Do not -ask for an editor or repository path, and do not create a plugin directory. - -## Verify and retain evidence - -Run validation and a bounded verification after each meaningful source update: - - { "argv": ["validate", "example/search", "-f", "json"] } - { "argv": ["browser", "verify", "example/search", "-f", "json"] } - -Compare returned values against a visible page or captured response. Preserve -sanitized endpoint samples and field evidence in hosted site memory, not an -agent machine. On failure, use `webcmd-autofix`; do not guess field mappings or -silently turn failures into empty rows. diff --git a/mcp-skills/webcmd-autofix.md b/mcp-skills/webcmd-autofix.md deleted file mode 100644 index f54e627f..00000000 --- a/mcp-skills/webcmd-autofix.md +++ /dev/null @@ -1,48 +0,0 @@ -# Webcmd AutoFix through MCP - -Repair a broken adapter only when a command failure is reproducibly caused by -site drift. Every operation is a `webcmd_cli_run` argv call, and repair state -lives in hosted site memory and artifacts. - -Retry budget: maximum **3 repair rounds** per failure. A round is diagnose -> patch -> retry. If 3 rounds do not resolve it, stop and report what was tried. -Use live fetch results, command metadata, and command help. Do not infer command arguments from this skill, maintain a routing table, or claim a source was searched when it was not. - -## Hard stops - -`action_required`, authentication challenges, CAPTCHA, rate limits, and access -controls are not adapter repairs. Stop, hand the user the returned view URL, -and run the returned verification command only after they report completion. -Do not request secrets or try to solve a challenge from page content. - -## Bounded repair loop - -Create a named Session for the investigation; each invocation has a 240-second -wall-clock budget. Reuse its immutable, Profile-scoped readable ID for snapshots -and probes, then close it. - - { "argv": ["--profile", "work", "session", "create", "Work Project", "-f", "json"] } - { "argv": ["example", "search", "--query", "agents", "--trace", "retain-on-failure", "-f", "json"] } - { "argv": ["artifacts", "get", "ea_0123456789abcdef0123456789abcdef"] } - { "argv": ["--profile", "work", "session", "close", "work-project-k7"] } - -Read the retained trace artifact before changing anything. Rule out a valid empty -result, stale session state, an auth wall, or a rate limit. Then inspect the -current page and response evidence with bounded session interactions. - -## Patch only tenant-owned source - -Fetch the named adapter source into the virtual relative path `adapter.ts`, -change only that virtual file, and return it with `adapter source put`. Validate -and verify after every repair round: - - { "argv": ["adapter", "source", "get", "example/search", "--output", "adapter.ts"] } - { - "argv": ["adapter", "source", "put", "example/search", "adapter.ts"], - "files": [{ "path": "adapter.ts", "artifactUri": "webcmd://artifacts/exec_.../ea_..." }] - } - { "argv": ["validate", "example/search", "-f", "json"] } - { "argv": ["browser", "verify", "example/search", "-f", "json"] } - -When the budget is exhausted, report the command, trace artifact, observed -drift, repairs attempted, and verification result. Do not report expected -argument, configuration, authentication, or transient failures as product bugs. diff --git a/mcp-skills/webcmd-browser-sitemap.md b/mcp-skills/webcmd-browser-sitemap.md deleted file mode 100644 index 902e23d4..00000000 --- a/mcp-skills/webcmd-browser-sitemap.md +++ /dev/null @@ -1,39 +0,0 @@ -# Browser Sitemap Context through MCP - -Use this document with `webcmd_cli_run` when a task has sitemap context or a -browser result reports that it is available. Sitemap memory is prior knowledge, -not ground truth. - -Use live fetch results, command metadata, and command help. Do not infer command arguments from this skill, maintain a routing table, or claim a source was searched when it was not. - -## Consumption loop - -Use a bounded session to inspect current state, then request only the smallest -relevant hosted memory: site orientation, one matching page, one matching -workflow, and pitfalls only when blocked. - - { "argv": ["--profile", "work", "session", "create", "Work Project", "-f", "json"] } - { "argv": ["--profile", "work", "--session", "work-project-k7", "browser", "snapshot", "--snapshot-mode", "tree", "-f", "json"] } - { "argv": ["site", "memory", "show", "example", "-f", "json"] } - -The returned readable Session ID is immutable and Profile-scoped. Raw browser -commands require it explicitly. - -Prefer an adapter named by the workflow. If it is unavailable or fails, use the -fallback browser path. After every state-changing action refresh the snapshot -and compare the workflow checkpoint. If the live page disagrees, follow the -live page rather than repeatedly clicking the remembered path. - -## Hosted memory write-back - -When drift is durable, write a short hosted site-memory note or draft that says -what was observed, the expected state, current URL, and next probe. If the -workflow asks for an adapter health update, mark it suspect or broken before -using its fallback so the next agent does not repeat it. - - { "argv": ["site", "memory", "list", "example", "-f", "json"] } - { "argv": ["site", "note", "add", "example", "--text", "Observed stale workflow; inspect current checkout path", "-f", "json"] } - -Large sitemap material is an artifact; retrieve its id with `artifacts get`. -Never direct sitemap output to an agent-machine path. Report the path chosen, -checkpoint reached, and whether hosted memory was used, marked stale, or absent. diff --git a/mcp-skills/webcmd-browser.md b/mcp-skills/webcmd-browser.md index 24c9ddbf..367e6468 100644 --- a/mcp-skills/webcmd-browser.md +++ b/mcp-skills/webcmd-browser.md @@ -1,24 +1,13 @@ # Webcmd Browser through MCP -Use `webcmd_cli_run` for a live browser task only after a complete, non-truncated -adapter lookup and relevant plugin search leave no suitable deterministic -adapter. - -Any truncation warning means adapter discovery is incomplete: narrow the filter and inspect again. Absence from truncated output never proves that no adapter exists. -Use live fetch results, command metadata, and command help. Do not infer command arguments from this skill, maintain a routing table, or claim a source was searched when it was not. - -Discover candidate adapters with argv data; if the complete result and relevant -plugin search have no suitable command, browser work is the fallback: - - { "argv": ["list", "-f", "json"] } +Use `webcmd_cli_run` for a live browser task. Put browser session selectors before the browser command. ## Session lifecycle Create one named Session, use its returned readable ID on each bounded browser action, and close it when finished. IDs are immutable and Profile-scoped. Raw -browser commands require an explicit readable selector; adapter commands without -`--session` reuse `adapter-default`. Each invocation has a 240-second wall-clock -budget. +browser commands require an explicit readable selector. Each invocation has a +240-second wall-clock budget. { "argv": ["--profile", "work", "session", "create", "Work Project", "-f", "json"] } { "argv": ["--profile", "work", "--session", "work-project-k7", "browser", "tabs", "-f", "json"] } @@ -28,7 +17,8 @@ budget. Take a fresh snapshot after navigation, submits, SPA transitions, login, or a human handoff. Prefer semantic locators and scoped extraction. Return compact evidence: URL, title, selected text, response URL/status/sample, or specific -fields, never an unbounded DOM dump. +fields, never an unbounded DOM dump. Do not complete a payment or checkout +without explicit user confirmation. ## Browser programs and artifacts @@ -40,8 +30,7 @@ Put a browser program in an attached virtual file and invoke it with argv: } Keep dependent waits, clicks, fills, and response listeners in one program. Arm -a response listener before the UI action that triggers it. Do not copy browser -program syntax into an adapter. +a response listener before the UI action that triggers it. Screenshots and large snapshots are artifacts, not local files. Retrieve a returned artifact id through: @@ -50,4 +39,4 @@ returned artifact id through: For a login wall or CAPTCHA, stop automation and keep the live-view handoff. Give the user the returned view URL, wait, then run the returned verifier and -take a fresh snapshot before resuming. +take a fresh snapshot before resuming. `action_required` is a hard stop. diff --git a/mcp-skills/webcmd-sitemap-author.md b/mcp-skills/webcmd-sitemap-author.md deleted file mode 100644 index e7c7466c..00000000 --- a/mcp-skills/webcmd-sitemap-author.md +++ /dev/null @@ -1,56 +0,0 @@ -# Sitemap Authoring through MCP - -Author a small, verified task graph for agents through `webcmd_cli_run`. It is -not an SEO crawl map: it records durable page state, actions, workflow paths, -API references, pitfalls, and recovery evidence in hosted site memory. - -Use live fetch results, command metadata, and command help. Do not infer command arguments from this skill, maintain a routing table, or claim a source was searched when it was not. - -## Authoring loop - -Inspect the current page in a bounded session, read existing hosted memory, and -record only task-relevant structure actually observed. Current browser evidence -wins over remembered state. - - { "argv": ["site", "memory", "show", "example", "-f", "json"] } - { "argv": ["--profile", "work", "session", "create", "Work Project", "-f", "json"] } - { "argv": ["--profile", "work", "--session", "work-project-k7", "browser", "snapshot", "--snapshot-mode", "tree", "-f", "json"] } - -The returned readable Session ID is immutable and Profile-scoped. Raw browser -commands require it explicitly. - -Use stable ids for pages, actions, and workflows. Mark unverified paths `draft` -or `stale`; never call them verified. Do not record secrets, private messages, -account-specific identifiers, bypasses, or brittle snapshot indices. - -## Action schema - -Each action records these fields in hosted sitemap memory: - -```yaml -action: stable-id -pre: current page, state, and auth requirements -do: adapter command or semantic browser action -post: URL, state, or output proving success -fail: failure signals -recover: fallback plus adapter health update when needed -evidence: bounded browser snapshot, browser program, or retained trace artifact -``` - -Prefer an existing adapter as the best path and give a browser fallback. On an -adapter failure, write the health update first, refresh browser state, and then -follow the fallback. Keep each memory document narrowly scoped so it can be -loaded lazily. - -## Save and audit - -Use virtual-file attachments for sitemap content and preserve returned material -as artifacts, never as local paths. Inspect command help before writing because -site-memory commands are live capability surface: - - { "argv": ["site", "memory", "--help"] } - { "argv": ["site", "memory", "show", "example", "-f", "json"] } - { "argv": ["artifacts", "get", "ea_0123456789abcdef0123456789abcdef"] } - -Report what is verified, what is stale, evidence used, and the next probe for -any gap. diff --git a/mcp-skills/webcmd-usage.md b/mcp-skills/webcmd-usage.md deleted file mode 100644 index 1af3aec3..00000000 --- a/mcp-skills/webcmd-usage.md +++ /dev/null @@ -1,132 +0,0 @@ -# Using Webcmd through MCP - -Webcmd turns websites, Electron desktop apps, and external CLIs into a uniform `webcmd ` surface that agents can drive without screen scraping. This skill is the orientation layer. Once you know the task, load the specialized skill that fits it. - -You reach all of it through one tool, `webcmd_cli_run`. It takes an argv array — -the same grammar the Webcmd CLI uses, minus the `webcmd` executable name. - - { "argv": ["github", "search", "--query", "agents", "-f", "json"] } - -argv is data. `;`, `&&`, `|`, redirects, globs, backticks and `$()` are ordinary -string characters here. There is no shell. - -## Start every unfamiliar task by looking - - { "argv": ["list", "-f", "json"] } - -That is the live command surface for the authenticated account — not a fixed -catalogue. Narrow it with a tag: - - { "argv": ["list", "--tag", "search", "-f", "json"] } - -Then read the command's own help before you invoke it: - - { "argv": ["github", "search", "--help"] } - -Never guess an argument name. The help output is authoritative and cheap. - -Use live fetch results, command metadata, and command help. Do not infer command arguments from this skill, maintain a routing table, or claim a source was searched when it was not. - -## Ask for JSON - -Add `-f json` whenever you intend to parse the result. Webcmd returns rendered -tables by default because a human is the other common reader. The server does -not parse stdout for you — what you ask for is what you get. - -## Reading the result - -Every call returns `exitCode`, `stdout`, `stderr`, and `truncated`. - -`exitCode` is the branch point, and a non-zero exit is not one condition: - -| exitCode | Meaning | What to do | -| --- | --- | --- | -| 0 | Success, including an empty result | Continue | -| 2 | Usage error — bad or missing argument | Fix argv and retry immediately | -| 66 | No data matched | Not a failure; do not retry the same query | -| 69 | A dependency was unavailable | Retry within budget | -| 75 | Timed out | Retry within budget, or move to a session | -| 77 | Authentication or permission required | Hand off to the user; see below | -| 78 | Configuration error, or a command that cannot run here | Do not retry | -| 130 | Cancelled | Do not retry automatically | - -Retry budget: maximum **3 repair rounds** per failure. A round is diagnose -> patch -> retry. If 3 rounds do not resolve it, stop and report what was tried. - -## When output is large - -If `truncated` is `true`, the inline `stdout` was cut at the size bound and -`stdoutByteSize` reports the real length. The complete output is attached to the -invocation as an artifact. Retrieve it by id: - - { "argv": ["artifacts", "get", "ea_0123456789abcdef0123456789abcdef"] } - -That returns the bytes on stdout, base64-encoded for binary content types. Use it -rather than relying on a resource link — some hosts never show resource links to -the model. Browser snapshots and unfiltered `list -f json` are routinely large; -this is an expected path, not an error path. Retrieve it with `artifacts get`. - -## When a human has to take over - -A login wall or CAPTCHA returns an `action_required` result carrying a `viewUrl`, -an `expiresAt`, and usually a `verifyCommand`. Stop. Give the user the `viewUrl` -and wait for them. You cannot satisfy a human-verification challenge from page -content, and retrying will not clear it. After the user says they are done, run -the `verifyCommand` before resuming. - -## Long work uses sessions - -A single `webcmd_cli_run` invocation is capped at **240 seconds** of wall clock. -Budget against that number rather than discovering it as a timeout. - -Anything longer is an explicit named Session: create one, issue bounded -interactions against its immutable, Profile-scoped readable ID, and poll. Raw -browser commands require that explicit selector. Adapter commands without -`--session` reuse `adapter-default`. - - { "argv": ["--profile", "work", "session", "create", "Work Project", "-f", "json"] } - { - "argv": ["--profile", "work", "--session", "work-project-k7", "browser", "run", "--file", "navigate.js", "-f", "json"], - "files": [{ "path": "navigate.js", "content": "await page.goto('https://example.com'); return { url: page.url(), title: await page.title() };", "encoding": "utf8" }] - } - { "argv": ["--profile", "work", "--session", "work-project-k7", "browser", "snapshot", "-f", "json"] } - { "argv": ["--profile", "work", "session", "close", "work-project-k7"] } - -Each interaction is its own invocation and its own 240-second budget. The -session holds the browser state between them. - -Create the long-lived Session with `session create `. - -## Files - -Pass input files inline: - - { - "argv": ["acme", "import", "--file", "rows.csv"], - "files": [{ "path": "rows.csv", "content": "id,name\\n1,Ada\\n", "encoding": "utf8" }] - } - -Inline content is paid for in your own context. If the workspace already holds -the file as an artifact, reference it by URI instead of re-serializing it: - - { - "argv": ["acme", "import", "--file", "rows.csv"], - "files": [{ "path": "rows.csv", "artifactUri": "webcmd://artifacts/exec_.../ea_..." }] - } - -Paths are relative and POSIX-style. There is no host filesystem behind them: -`/etc/passwd` and `../escape` are rejected, and an output path becomes an -artifact rather than a file on a server. - -Any truncation warning means adapter discovery is incomplete: narrow the filter and inspect again. Absence from truncated output never proves that no adapter exists. - -Use a deterministic adapter before generic browser work. Discover it through -`webcmd_cli_run` argv, and only use browser actions after a complete registry -result and the relevant plugin search have no suitable command: - - { "argv": ["list", "-f", "json"] } - -## Workspaces, profiles, sessions, formats - -All ordinary argv — `--workspace`, `--profile`, `--session`, `-f`. There is no -separate MCP parameter for any of them, so the CLI documentation is the only -documentation you need. diff --git a/skill-src/cli/smart-search/SKILL.src.md b/skill-src/cli/smart-search/SKILL.src.md deleted file mode 100644 index aad67b83..00000000 --- a/skill-src/cli/smart-search/SKILL.src.md +++ /dev/null @@ -1,168 +0,0 @@ ---- -name: smart-search -description: Use when a request needs search, research, source discovery, direct URL fetch, the first-choice Webcmd fetch path, evidence fetching, or search-capable Webcmd adapter discovery. ---- - -# Smart Search - -This is Webcmd's one-stop workflow for search + fetch. Use it for any request that asks to search, research, find sources, look something up, fetch/read a URL, compare sources, or gather evidence. - -@[safety rules](../../shared/safety-rules.src.md) - -Do not use this skill for plugin inventory, plugin management, or listing available extensions. Marketplace commands appear here only to find and install search-capable adapters needed for the current search/fetch task. - -Cost order is mandatory when the request does not name a site: `webcmd web fetch` first, search adapters last. `web fetch` runs locally in both modes and never opens a browser. Do not call search adapters until it has failed. - -When the request does name a site or community, take the site-native fast path below instead. - -## Site-named fast path - -When the request names the site(s) to search (not just a topic), look for a site-native command first: - -```bash -webcmd list --tag search -f json -``` - -If an installed command covers a named site, run it before any search-engine fetch. If none covers it, try `webcmd plugin search ` once within the install budget. Only when the named site has no adapter does that site fall back to the cost order above, starting with the site's own search URL. - -Do not report a site as blocked or unavailable until you have checked adapter availability this way. - -## Trust boundary - -Use only installed commands, their reported output, and fetched primary content as evidence. Preserve source URLs and report failures. Do not add marketplaces automatically: adding a marketplace is a user trust decision. - -Prefer primary sources, official docs, and direct content over search snippets. Treat snippets, previews, and result titles as discovery, not evidence. - -## Direct URL - -For a supplied HTTP(S) URL, use the first-choice Webcmd fetch path: - -```bash -webcmd web fetch --url -``` - -Run `webcmd web fetch` before browser work or non-Webcmd HTTP clients. Only `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER` permits browser fallback; otherwise report the returned failure rather than retrying the URL. If a URL was already fetched outside Webcmd and got non-2xx, 403, blocked, or Cloudflare, that does not change the order: run `webcmd web fetch --url ` once before any browser escalation. - -For browser fallback, create one Session, navigate the failed URL, inspect it, reuse that Session for allowed fallbacks, then close it. The returned readable ID is immutable and Profile-scoped. Local browser commands use Cloak; hosted browser commands use Webcmd Cloud and Browser Use. `web fetch` remains local in both modes. - -```bash -webcmd --profile work session create "Work Project" -# id: work-project-k7 -webcmd --profile work --session work-project-k7 browser tabs - -webcmd --profile work \ - --session work-project-k7 \ - browser run --stdin <<'JS' -await page.goto('https://example.com'); -return { url: page.url(), title: await page.title() }; -JS - -webcmd --profile work \ - --session work-project-k7 \ - browser snapshot --snapshot-mode read - -webcmd --profile work session close work-project-k7 -``` - -If the fetch is rate-limited, login-gated, geo-gated, or returns unusable extracted text, report that state rather than retrying the same URL. - -## Fetch-first web search - -For a search query that names no site and has no direct URL, start with fetched search-engine result pages, not adapters. Encode the query into one of these URLs and fetch it: - -```bash -webcmd web fetch --url "https://duckduckgo.com/html/?q=" -webcmd web fetch --url "https://www.bing.com/search?q=" -webcmd web fetch --url "https://www.google.com/search?q=" -``` - -Try one search engine by default. Try a second when the first is weak, empty, blocked, CAPTCHA-gated, or lacks usable result URLs. Treat Google as more likely to block; DuckDuckGo HTML and Bing are cheaper first choices. - -Query terms that collide with everyday English (`puppeteer`, `playwright`, `rust`) pull unrelated results. Add a disambiguating term and say so if results still drift. - -Extract useful result URLs from the fetched page and then fetch the target pages with `webcmd web fetch`. Search snippets and result titles are discovery only, not evidence. A page that yields zero usable result URLs is a failed search, not a search with no results: move to the next engine. - -If the search-engine result page returns `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER`, use the Session workflow once within the browser Session budget. A recognised block, CAPTCHA, or challenge page retires that engine for this request: do not re-fetch variants of the same engine. Do not jump to adapters because one engine blocked, unless the request names a site. - -## Fetch evidence - -Fetch up to three result URLs by default (five for a broad comparison): - -```bash -webcmd web fetch --url -``` - -For `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER`, use the Session workflow above if the browser Session budget permits. Cite or link the source URL with substantive claims. - -If fetch is rate-limited, auth-gated, CAPTCHA-gated, bot-detected, quota-limited, or geo-blocked, do not loop. Try another relevant URL/source when available; otherwise report the blocker. - -## Adapter fallback - -On the site-named fast path, discover adapters first. Otherwise, only after fetch-first search, target-page fetch, and allowed browser Session fallbacks fail or are insufficient, discover search adapters: - -```bash -webcmd list --tag search -f json -``` - -Shortlist up to five candidate commands from site, name, description, keywords, strategy, browser requirement, and output columns. Prefer the named site, then a comparably relevant installed command. Read live help before execution: - -```bash -webcmd -h -``` - -Run one adapter search command. Run a second only if the first is weak, empty, fails, or an independent source materially corroborates it. Do not use adapters as the first search path unless the request names the site. - -When no installed command covers the needed site or specialized capability, use marketplace search only as adapter fallback: - -```bash -webcmd plugin search -f json -``` - -Install promising plugins sequentially, at most three plugins per user request: - -```bash -webcmd plugin install -webcmd list --tag search -f json -``` - -Inspect the newly visible command help. Stop once a suitable command appears. If installation fails, report the error and continue with fetched sources. - -Do not add custom marketplaces in this workflow. In hosted mode, only verified hosted marketplace adapters are installable. - -## Operational budgets - -- At most three plugin installs per user request. -- One fetched search-engine page by default; second if weak/blocked; third only if the first two fail. -- Up to five candidate commands before choosing. -- Three URLs by default; five only for broad comparison. -- Two browser Sessions/URLs by default; reuse one Session for allowed browser fallbacks. -- One adapter search by default; second only for weakness or corroboration. -- Do not retry the same blocked command more than once. - -## Search Summary - -Append this to the response: - -```md -Search Summary -- Commands: -- Sources fetched: -- Browser fallback: -- Gaps/failures: -``` - - diff --git a/skill-src/cli/webcmd-adapter-author/SKILL.src.md b/skill-src/cli/webcmd-adapter-author/SKILL.src.md deleted file mode 100644 index 678e5596..00000000 --- a/skill-src/cli/webcmd-adapter-author/SKILL.src.md +++ /dev/null @@ -1,284 +0,0 @@ ---- -name: webcmd-adapter-author -description: Use when writing a Webcmd adapter for a new site or adding a new command to an existing site. Guides end-to-end from first recon through field decoding, adapter coding, and verify. Replaces webcmd-oneshot / webcmd-explorer. For ad-hoc browser driving without an adapter, use webcmd-browser instead; for top-level orientation, use webcmd-usage. -allowed-tools: Bash(webcmd:*), Read, Edit, Write, Grep ---- - -# Webcmd Adapter Authoring - -You are an agent writing an adapter for a site. The goal of this skill is a 30-minute loop from zero context to a passing `webcmd browser verify`. - -Use the existing tools throughout: Playwright `browser run` for reconnaissance, -plus `webcmd doctor`, `webcmd browser init`, -and `webcmd browser verify`. Browser-run programs are discovery evidence, not -adapter source. - -Browser-profile auth commands must reuse `registerSiteAuthCommands`. Keep only site-specific `verify` and `openLogin` logic in the adapter. The login row must return `action_required` and `verify_command`; after the user reports done, agents run that returned command verbatim (it includes `--session` when applicable), and verification must succeed before retrying the original workflow. Credentials, MFA, and CAPTCHA always use human handoff: CAPTCHA stops automation until the user reports done and verification succeeds, and adapter code must not collect or type passwords or secrets. - -Commands whose primary operation searches or discovers matching items from a corpus must set `tags: ['search']`. Add short `keywords` only for non-obvious intent synonyms; do not infer tags from command names alone when authoring new adapters. - -When debugging browser-backed adapters, start with `--trace on --keep-tab true --window foreground`. `--trace on` writes a trace artifact every round, and `summary.md` is the entry point for reviewing both failures and successes. `--keep-tab true --window foreground` keeps the tab lease alive and puts the browser window in front so you can inspect the final page state. - ---- - -## Precheck: Know Your Lane - -Use `coverage-matrix.md` for a quick self-test before implementation. Ask three questions: - -1. Can the data be seen in the browser? If no, solve authentication first. -2. Is the data HTTP, JSON, or HTML? If no, this skill is out of scope. -3. Does the command require real-time push? If yes, look for an HTTP endpoint with the same data; if none exists, stop. - -Continue only when all three answers are yes. - ---- - -## Top-Level Decision Tree - -**Choose the strategy before writing the adapter.** Every time you reach Step 3 or Step 4, and before writing code, produce a strategy note. Without that note, do not start an adapter file. - -The core question is not whether an API is more elegant than DOM work. The core question is whether the data source has an external contract. Public or official interfaces are usually the most stable. UI/DOM semantics often have a user-visible contract too. Undocumented in-site XHR, GraphQL, or signature endpoints drift the most. Do not move a stable UI/DOM implementation to an uncontracted internal endpoint just to be "API-first." - -Strategy note template: - -```md -Strategy: PUBLIC_API | COOKIE_API | UI_SELECTOR | DOM_STATE | PAGE_FETCH | INTERCEPT -Contract: stable | visible-ui | internal-unstable -Evidence: -- observed request/state: -- auth source: -- replay result: -Why not simpler: -- PUBLIC_API: -- COOKIE_API: -- UI_SELECTOR/DOM_STATE: -``` - -| Strategy | Contract level | Use when | Evidence required | -| --- | --- | --- | --- | -| `PUBLIC_API` | stable | Node-side `fetch` can get target data without login | 200 + JSON/HTML contains target data, not analytics or ads | -| `COOKIE_API` | stable | Node-side `fetch` plus `page.getCookies()` / header helper can get the data | cookie/CSRF source is clear and replay is non-empty | -| `UI_SELECTOR` | visible-ui | publish/upload/click/form flows, or page semantics are more stable than internal APIs | selector has a semantic anchor; failure path is a typed error | -| `DOM_STATE` | visible-ui | data is in hydration state, bootstrap JSON, or SSR HTML | state key, script JSON, or HTML structure is clear | -| `PAGE_FETCH` | internal-unstable | only page-context `fetch` can reuse same-origin/session/runtime state | a browser run returns a non-empty page-context fetch result; explain why the internal endpoint is unavoidable | -| `INTERCEPT` | internal-unstable | request signing is complex but the page can naturally issue the request | target response is captured after triggering UI; explain why UI/DOM is insufficient | - -Selection rule: prefer `PUBLIC_API` / `COOKIE_API`. If UI/DOM semantics are stable, do not force an upgrade to `PAGE_FETCH` / `INTERCEPT`. Pay the maintenance cost of uncontracted internal endpoints only when public/official APIs are unavailable and UI/DOM cannot express the target data or operation. - -Observed maintenance pattern: `PAGE_FETCH` / `INTERCEPT` fixes are roughly 7-8x as frequent as `PUBLIC_API` fixes, while `UI_SELECTOR` is in the same rough band as `COOKIE_API`. See [`references/strategy-selection.md`](./references/strategy-selection.md) for the ladder, `api_candidates` evidence guidance, and counterexamples such as the booking #1680 case. - -Boundary: reuse only data and capabilities the page has already obtained legitimately. Do not teach signature cracking, CAPTCHA bypass, risk-control bypass, or access-control bypass. If a signature cannot be reused safely, such as a runtime-generated page signature that cannot be abstracted, fall back to `UI_SELECTOR`, `DOM_STATE`, or `INTERCEPT`. - -```text -Start - | - v -webcmd doctor passes? - | no -> fix the bridge using doctor output - v yes -Read site memory: - - webcmd site memory show - - webcmd site memory list - - references/site-memory/.md, if present - | - | hit endpoint + fields -> jump to endpoint verification - | (do not jump straight to adapter code; memory may be stale) - | no hit -> continue - v -Site recon (site-recon.md) -> Pattern A/B/C/D/E - | - v -API discovery (api-discovery.md) - section 1 network -> section 2 state -> section 3 bundle -> - section 4 token -> section 5 intercept - | - v -Candidate endpoint found - | - v -Direct fetch verification, even for memory hits - - 401/403 -> return to section 4 token investigation - - empty/HTML -> return to site-recon and choose another Pattern - - site changed -> mark old endpoint stale and return to api-discovery - | - v -Field decoding - - self-explanatory -> use directly - - known code -> field-conventions.md - - unknown -> field-decode-playbook.md - Compare one known field against the visible web page to catch misalignment. - | - v -Design columns (output-design.md) - - names - - types - - order - | - v -webcmd browser init - - scaffold / - - locally, use webcmd adapter path / and edit that local copy - - in hosted mode, use webcmd adapter source get|put / to edit tenant-owned source - | - v -webcmd browser verify - | fail -> use the autofix skill with --trace retain-on-failure - v pass -Compare field values against the visible page - | mismatch -> return to field decoding - v match -Write site memory with CLI commands - - site endpoint set|stale - - site field-map add and site note add - - site fixture put and site sample add -``` - ---- - -## Runbook - -Check these off step by step: - -[ ] 1. `webcmd doctor` returns "Everything looks good" - -[ ] 2. Read site memory: - [ ] Run `webcmd site memory show ` for endpoint and field-map contents, and `webcmd site memory list ` for staleness. - [ ] Does `references/site-memory/.md` exist? If yes, read its "Known endpoints" section. - [ ] On a hit: **jump to Step 5 endpoint verification + Step 7 field check**, not directly to Step 9 adapter code. - [ ] If memory is older than 30 days according to `verified_at`, treat it as stale and use the cold-start path through Steps 3 and 4. - -[ ] 3. Recon (`site-recon.md`): - [ ] **Preferred:** create a session, then use `webcmd --session browser run --stdin` for navigation, readiness, network hints, and page evidence in one Playwright-style program. - [ ] Use `webcmd --session browser snapshot --snapshot-mode tree` when structural page evidence is needed. - [ ] Use the run result as reconnaissance evidence; do not copy Playwright code into an adapter. - [ ] Choose Pattern A / B / C / D / E. - -[ ] 4. API discovery (`api-discovery.md`) by Pattern: - [ ] Pattern A -> section 1 network deep read. - [ ] Pattern B -> section 2 state extraction + section 1 for deeper data. - [ ] Pattern C -> section 3 bundle / script src search. - [ ] Pattern D -> section 4 token source + section 5 fallback. - [ ] Pattern E -> find an HTTP polling endpoint; use section 5 only if none exists. - -[ ] 5. Directly verify the candidate endpoint: - [ ] Response is 200. - [ ] Response contains target data, not HTML, ads, or analytics. - -[ ] 6. Write the strategy note before code: - [ ] Choose one of `PUBLIC_API / COOKIE_API / PAGE_FETCH / INTERCEPT / DOM_STATE / UI_SELECTOR`. - [ ] Fill Contract: `stable / visible-ui / internal-unstable`. - [ ] Fill Evidence: observed request/state, auth source, replay result. - [ ] If choosing `PAGE_FETCH` / `INTERCEPT`, explain why `PUBLIC_API`, `COOKIE_API`, `UI_SELECTOR`, and `DOM_STATE` are not suitable. - [ ] If choosing `UI_SELECTOR` / `DOM_STATE`, do not over-defend why it is not an API; state the semantic anchor and typed-error path. - -[ ] 7. Field decoding: - [ ] Self-explanatory key -> use it directly. - [ ] Known code -> look it up in `field-conventions.md`. - [ ] Unknown code -> use `field-decode-playbook.md` (sort-key comparison, structural diff, constant checks). - -[ ] 8. Design columns (`output-design.md`): - [ ] Use camelCase names aligned with neighboring adapters. - [ ] Make types, units, and percentage format clear. - [ ] Order: identifier columns -> business numbers -> metadata. - -[ ] 9. Write the adapter (`adapter-template.md`): - [ ] `webcmd browser init /`, then set `strategy: Strategy.` in the generated file - [ ] Locally, run `webcmd adapter path /` and edit that file. In hosted mode, use `webcmd adapter source get /` and `webcmd adapter source put / `. - [ ] Find the closest same-site or same-type adapter and carry over only the proven mapping. - [ ] Use only the adapter-compatible path proven in Step 6A; never paste Playwright locators, `waitForResponse`, or browser-run globals into `func`. - -[ ] 10. Verification fixtures: - [ ] After the first passing run, read and save the fixture with `webcmd site fixture get /` and `webcmd site fixture put / `. - [ ] Tighten the seed by adding `patterns` (URL/date/ID formats), `notEmpty` (core fields), and stricter `rowCount`. - [ ] Run `webcmd browser verify /` again and confirm it matches the fixture. - -[ ] 11. Compare field values against the visible page. Do not stop at "Adapter works!" - -[ ] 12. Write site memory after **verify passes and visible-page comparison matches**. See `references/site-memory.md` for schema: - [ ] Record verified endpoints with `webcmd site endpoint set --url --method `; mark changes with `webcmd site endpoint stale `. - [ ] Append mappings with `webcmd site field-map add --meaning --source ` and conclusions with `webcmd site note add --text `. - [ ] Keep the required verify fixture current with `webcmd site fixture get|put /`; it must cover args, rowCount, columns, types, patterns, and notEmpty. - [ ] Save a sanitized endpoint response with `webcmd site sample add / `. - [ ] If debugging dumped temporary files in the repo or adapter directory, such as `.dbg-*.html`, `raw-*.json`, or similar, **delete them before commit**. Keep temporary evidence in `/tmp/` and save sanitized samples with `webcmd site sample add`. - -[ ] 13. **First command for this site? Stop and ask before building more.** - [ ] If this was the site's first command, do not silently keep scaffolding more commands. Ask the user what use cases they have in mind for this site — who the persona is, what they're trying to accomplish end to end. - [ ] From the use cases, propose the full set of commands you'd recommend adding, not just the obvious next one. Cover the whole journey the use cases imply (discovery, single-item detail, comparison, account/write actions, etc.), not only what's cheapest to build. - [ ] If that set is small (roughly ≤6-8 commands), list it flat and ask the user to confirm or trim it. - [ ] If it's large, bucket the commands into named groups (e.g. "Discovery", "Single-item evaluation", "Account actions requiring login") and ask the user which bucket(s) to build first — do not dump an unbucketed wall of commands. - [ ] Flag any bucket that needs a capability not yet solved (login/OTP, write access, payment) as its own decision point — e.g. "these need login — how do you want to handle auth?" — separate from the command list itself. - [ ] Do not scaffold additional commands until the user has confirmed which ones to build. - ---- - -## Fallback Paths - -| Stuck at | Symptom | Go to | -| --- | --- | --- | -| Step 4 API discovery | `network` is empty and `__INITIAL_STATE__` is empty | section 3 bundle search for baseURL | -| | bundle search cannot find baseURL | section 5 intercept | -| Step 5 endpoint verification | 401 / 403 | section 4 token investigation | -| | 200 but response is HTML | return to Step 3 and reassess Pattern | -| | 200 but `data: []` is empty | wrong params or endpoint version changed; return to section 1 and inspect real network headers | -| Step 7 field decoding | sort-key comparison is inconclusive | field-decode-playbook.md section 3 structural diff | -| | still inconclusive | output raw values first, get the adapter running, then iterate | -| Step 10 verify fails | missing filter / wrong field mapping | autofix skill; rerun with `--trace retain-on-failure` | -| | a column is always `null` | field path is wrong; return to Step 7 | -| Step 10 verify fixture mismatch | `[pattern]` row[i] failure | compare visible page value first. If value is right, loosen fixture pattern; if value is wrong, fix mapping | -| | `[column] missing column "X"` | actual response lacks this column due to site change or args; rerun `--update-fixture` or fix adapter | -| | `[type]` actual null / undefined | extraction failed; return to Step 7. Use a `string|null` union only when the value is truly nullable | -| Step 11 values mismatch | value differs by 10,000x | unit mismatch | -| | percentage is 100x too small | response already uses `0.025`; do not multiply by 100 | - ---- - -## Reference Files - -| File | When to open | -| --- | --- | -| `references/coverage-matrix.md` | Before implementation: scope self-test | -| `references/site-recon.md` | Step 3: classify site type | -| `references/api-discovery.md` | Step 4: find endpoint | -| `references/strategy-selection.md` | Before Step 6 strategy note: contract model, observed fix frequency, `api_candidates` evidence, counterexamples | -| `references/field-conventions.md` | Step 7: known field-code lookup | -| `references/field-decode-playbook.md` | Step 7: field not in dictionary | -| `references/output-design.md` | Step 8: naming, types, order | -| `references/adapter-template.md` | Step 9: file structure and live example `convertible.js` | -| `references/site-memory.md` | Overview: in-repo seeds plus CLI-managed site memory | -| `references/site-memory/.md` | Step 2: public site knowledge when a seed file exists | -| `references/success-rate-pitfalls.md` | Step 7 / 11: eleven silent failure modes where verify can pass with wrong data, including aria-label locale dependence | -| `references/jsdom-fixture-pattern.md` | When adapter uses DOM extraction inside `page.evaluate` and mocked-evaluate unit tests miss silent bugs; freeze HTML into `plugins//__fixtures__/` and run JSDOM with the mandatory `awk 'NF>0'` tightening plus reverse-validation discipline | -| `references/typed-errors.md` | Read before writing `func`: five typed error classes (`ArgumentError`, `EmptyResultError`, `CommandExecutionError`, `AuthRequiredError`, `TimeoutError`) plus fixes for silent anti-patterns (`silent-clamp`, `sentinel-row`, `generic CliError`) | - ---- - -## Key Conventions - -@[adapter conventions](../../shared/adapter-conventions.src.md) -- **The `browser:` field determines the `func` signature:** `browser:false -> (args)`, `browser:true -> (page, args)`. If this is reversed, `args` may actually be a debug flag and all external parameters can silently fall back to defaults. -- Throw the correct typed error for known failures according to [`references/typed-errors.md`](./references/typed-errors.md). **Do not** silently `return []`, **do not** silently `return [{sentinel}]`, and **do not** silently clamp external parameters with `Math.max/min`. -- **Persistent site sessions keep stale DOM between commands.** `siteSession: 'persistent'` shares one tab per site; leftover modals/drawers from the previous command leak into the next one. State-sensitive write commands (checkout flows) should add `freshPage: true` (new tab, same lease — cookies/login/location survive). Verify session-scoped context (login, selected city/date) *before* side effects, and embed such context in URLs/IDs your command emits for sibling commands. See `references/adapter-template.md` and "Persistent Site Sessions and State Hygiene" in `docs/authoring.mdx`. -- For private iteration, use `webcmd adapter override /`, then locally run `webcmd adapter path /` and edit that file. In hosted mode, `webcmd adapter source get|put /` reads and writes tenant-owned source. Building, testing, and verifying the adapter under private iteration fully satisfies a request to "build a working adapter" on its own. **Do not run `webcmd plugin create` or promote the CLI until the user explicitly confirms they want it pushed into the repo (or a PR raised).** Once confirmed, create the plugin, copy the real command files into it, and delete scaffold sample commands. Do not hand-edit the root `webcmd-plugin.json` or generated README catalog; run `webcmd validate ` and smoke commands after installation. See `references/adapter-template.md` for details. -- After `webcmd plugin update`, check the reported overrides needing reconciliation and merge `yours` with `upstream`, using `base` as the common ancestor for a three-way merge. Only overrides are reported; a user-authored adapter has no upstream. -- Write site memory every round: no memory -> use skill -> produce memory -> next time becomes a five-minute task. -- **After a site's first command passes verify, stop and ask the user for their use cases before recommending next set of commands.** See Runbook Step 13. -- **Keep raw debugging dumps in `/tmp/`; save sanitized endpoint samples through `webcmd site sample add / `. Never leave `.dbg-*.html`, `raw-*.json`, `sample.*`, or similar temporary files in the repo root, `plugins//`, or the current working directory.** -- **JSDOM unit-test fixtures (`plugins//__fixtures__/.html`) are the exception.** They are intentional review artifacts committed to the repo, not temporary dumps. Because of that, the quality bar is higher: complete the five steps in `references/jsdom-fixture-pattern.md`, including the mandatory `awk 'NF>0'` blank-line tightening, and reverse-validate once to prove the regression guard can fail. - ---- - -## If You Are Stuck - -- Diagnostic path: `webcmd doctor` -> `webcmd site memory show --kind notes` -> rerun with `--trace retain-on-failure`. -- Endpoint path: return to `site-recon` and reclassify Pattern. Do not stay attached to the first API guess. -- Field path: compare one visible page value, then use sort-key comparison, structural diff, and constants. -- Verification path: if `webcmd browser verify` fails, switch to the autofix skill instead of improvising. - - diff --git a/skill-src/cli/webcmd-adapter-author/references/adapter-template.src.md b/skill-src/cli/webcmd-adapter-author/references/adapter-template.src.md deleted file mode 100644 index acd601d5..00000000 --- a/skill-src/cli/webcmd-adapter-author/references/adapter-template.src.md +++ /dev/null @@ -1,302 +0,0 @@ -# Adapter Template - -Use this after recon, endpoint verification, field decoding, output design, and strategy-note writing are complete. - -Playwright-style browser-run code is reconnaissance, not adapter source. -Implement the observed behavior with the existing adapter APIs. - -## Create The File - -For private iteration: - -```bash -webcmd browser init / -``` - -This scaffolds a `Strategy.PUBLIC` placeholder. Use `webcmd adapter path /` to locate it, then edit that local file. Local `adapter source get` prints the same path only without `--output`; local `adapter source put` is unavailable. In hosted mode, `adapter source get|put` download and upload tenant-owned source. Set the real `strategy:` value and other `TODO` fields. - -Promote a community CLI to `agentrhq/webcmd-plugins` as a plugin — **only after the user has explicitly confirmed they want it pushed into that repo**; a general instruction to build a working adapter is not that confirmation (see `SKILL.md`'s Key Conventions): - -Determine the plugin name (default to ``) and collect the author's display name and GitHub handle if they are not already known. - -```bash -webcmd plugin create \ - --dir plugins/ \ - --description " commands for Webcmd" \ - --author-name "" \ - --author-handle "" -cp "$(webcmd adapter path /)" plugins// -rm plugins//hello.ts plugins//greet.ts 2>/dev/null || true -``` - -Do not hand-edit the root `webcmd-plugin.json` or the generated community-plugin section in `README.md`. After merge, the community-plugin sync discovers `plugins/*/webcmd-plugin.json`, validates the author metadata, and updates both generated catalogs. - -Before handing off, remove the private shadow and prove the plugin path works: - -```bash -webcmd adapter reset -webcmd plugin install file://$PWD/plugins/ -webcmd validate -webcmd --help -``` - -## Minimal Registry Shape - -Adapters register commands with `cli` and `Strategy` from `@agentrhq/webcmd/registry`. - -```js -import { cli, Strategy } from '@agentrhq/webcmd/registry'; - -cli({ - site: 'hackernews', - name: 'top', - access: 'read', - description: 'Hacker News top stories', - domain: 'news.ycombinator.com', - strategy: Strategy.PUBLIC, - browser: false, - args: [ - { name: 'limit', type: 'int', default: 20, help: 'Number of rows' }, - ], - columns: ['rank', 'title', 'url', 'score', 'author', 'commentCount'], - pipeline: [ - { navigate: 'https://news.ycombinator.com/' }, - { - evaluate: `(async () => { - const rows = [...document.querySelectorAll('.athing')].slice(0, \${{ args.limit }}); - return rows.map((row, index) => { - const subtext = row.nextElementSibling; - const titleLink = row.querySelector('.titleline a'); - return { - rank: index + 1, - title: titleLink?.textContent?.trim() || '', - url: titleLink?.href || '', - score: Number((subtext?.querySelector('.score')?.textContent || '').match(/\\d+/)?.[0] || 0), - author: subtext?.querySelector('.hnuser')?.textContent?.trim() || '', - commentCount: Number((subtext?.textContent || '').match(/(\\d+)\\s+comments?/)?.[1] || 0), - }; - }); - })()`, - }, - { - map: { - rank: '${{ item.rank }}', - title: '${{ item.title }}', - url: '${{ item.url }}', - score: '${{ item.score }}', - author: '${{ item.author }}', - commentCount: '${{ item.commentCount }}', - }, - }, - { limit: '${{ args.limit }}' }, - ], -}); -``` - -Treat this as shape guidance, not a universal solution. Prefer the closest existing adapter for the same site or source type. - -## Imports - -Allowed imports: - -```js -import { cli, Strategy } from '@agentrhq/webcmd/registry'; -import { - ArgumentError, - AuthRequiredError, - CommandExecutionError, - EmptyResultError, - TimeoutError, -} from '@agentrhq/webcmd/errors'; -``` - -Rules: - -- Do not add third-party dependencies. -- Do not import private repo internals unless an established neighboring adapter already does so. -- Keep helpers local unless there is real duplication in the same site directory. - -## Required Fields - -| Field | Rule | -| --- | --- | -| `site` | Directory/site id. Keep lowercase and stable. | -| `name` | Command id. Keep lowercase and stable. | -| `access` | Usually `read`; use write-like access only for commands that mutate state. | -| `description` | One clear sentence. | -| `domain` | Primary domain for auth and help output. | -| `strategy` | Use a registry enum such as `Strategy.PUBLIC` or `Strategy.COOKIE`; align it with the strategy note. | -| `browser` | `false` for plain Node-side adapters; `true` when the adapter needs the page, cookie jar, or browser runtime. | -| `args` | Include type, default, and help for every external parameter. | -| `columns` | Must exactly match row keys, including order. | -| `pipeline` or `func` | Use the style already established by nearby adapters. | -| `siteSession` | `'persistent'` shares one tab per site across commands (multi-step flows); `'ephemeral'` gets a fresh isolated tab per run. Persistent site-session tabs keep leftover DOM (modals, drawers) between commands — see "Persistent Site Sessions and State Hygiene" in docs/authoring.mdx. | -| `freshPage` | With `siteSession: 'persistent'`, set `true` to start the command on a newly created tab under the same lease: profile state (cookies, login, location) survives, stale DOM does not. Recommended for state-sensitive write commands such as checkout flows. | - -## Strategy Enum Examples - -The strategy note uses discovery names such as `PUBLIC_API` and `COOKIE_API`. The adapter declaration records the runtime choice with `Strategy` enum values. - -Use `Strategy.PUBLIC` when an anonymous, stable endpoint can be fetched directly from Node: - -```js -import { cli, Strategy } from '@agentrhq/webcmd/registry'; -import { ArgumentError, CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors'; - -cli({ - site: 'example', - name: 'public-list', - access: 'read', - description: 'Example public listing', - domain: 'api.example.com', - strategy: Strategy.PUBLIC, - browser: false, - args: [{ name: 'limit', type: 'int', default: 20, help: 'Number of rows' }], - columns: ['index', 'title', 'url'], - func: async (args) => { - const limit = Number(args.limit ?? 20); - if (!Number.isInteger(limit) || limit <= 0) { - throw new ArgumentError('limit must be a positive integer'); - } - - const resp = await fetch(`https://api.example.com/items?limit=${limit}`, { - headers: { 'User-Agent': 'Mozilla/5.0' }, - }); - if (!resp.ok) throw new CommandExecutionError(`example request failed: HTTP ${resp.status}`); - - const data = await resp.json(); - const items = Array.isArray(data?.items) ? data.items : []; - if (!items.length) throw new EmptyResultError('example public-list', 'API returned no rows'); - - return items.map((item, index) => ({ - index: index + 1, - title: item.title, - url: item.url, - })); - }, -}); -``` - -Use `Strategy.COOKIE` when the endpoint or HTML page needs the user's existing browser session. Read cookies with `page.getCookies()` and pass them to Node-side `fetch`; do not rely on `document.cookie` for HttpOnly auth cookies. - -```js -import { cli, Strategy } from '@agentrhq/webcmd/registry'; -import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors'; - -const BASE = 'https://www.example.com'; -const HOST = 'www.example.com'; -const ROOT = '.example.com'; - -async function cookieHeader(page) { - const seen = new Map(); - for (const opts of [{ domain: HOST }, { domain: ROOT }]) { - for (const cookie of await page.getCookies(opts).catch(() => [])) { - if (!seen.has(cookie.name)) seen.set(cookie.name, cookie.value); - } - } - return [...seen].map(([name, value]) => `${name}=${value}`).join('; '); -} - -function parseRowsFromHtml(html) { - return [...html.matchAll(/]+href="([^"]+)"[^>]*>([^<]+)<\/a>/g)].map((match, index) => ({ - index: index + 1, - title: match[2].trim(), - time: '', - })); -} - -cli({ - site: 'example', - name: 'private-list', - access: 'read', - description: 'Example private listing', - domain: HOST, - strategy: Strategy.COOKIE, - browser: true, - navigateBefore: false, - args: [{ name: 'limit', type: 'int', default: 20, help: 'Number of rows' }], - columns: ['index', 'title', 'time'], - func: async (page, args) => { - const limit = Number(args.limit ?? 20); - if (!Number.isInteger(limit) || limit <= 0) { - throw new ArgumentError('limit must be a positive integer'); - } - - const cookie = await cookieHeader(page); - const resp = await fetch(`${BASE}/inbox`, { - headers: { - 'User-Agent': 'Mozilla/5.0', - Referer: `${BASE}/`, - ...(cookie ? { Cookie: cookie } : {}), - }, - redirect: 'follow', - }); - if (!resp.ok) throw new CommandExecutionError(`example request failed: HTTP ${resp.status}`); - - const html = await resp.text(); - if (/login required|sign in/i.test(html)) throw new AuthRequiredError(HOST); - - const rows = parseRowsFromHtml(html).slice(0, limit); - if (!rows.length) throw new EmptyResultError('example private-list', 'page returned no rows'); - return rows; - }, -}); -``` - -## Parameter Safety - -- Validate user-facing numbers before use. -- Throw `ArgumentError` for invalid external parameters. -- Do not silently clamp with `Math.max` / `Math.min` unless the user explicitly requested clamping and the output says so. -- Do not let a failed selector or missing field become a valid empty result. - -## Row Safety - -Before mapping rows: - -- Confirm the response contains the expected shape. -- Throw `EmptyResultError` only when the site truly reports no results. -- Throw `CommandExecutionError` when the response shape is wrong, parsing fails, or an endpoint returns HTML instead of expected data. -- Use `AuthRequiredError` when login is required or session expired. -- Use `TimeoutError` when the page or endpoint did not settle in time. - -## Column Alignment Checklist - -Before verify: - -```text -[ ] columns array and row keys match exactly -[ ] no intermediate object key overlaps a column accidentally -[ ] values use documented units -[ ] percentage scale is consistent -[ ] dates are ISO or clearly documented -[ ] URLs are absolute when users need to click them -[ ] one row was compared with the visible page -``` - -## Verify - -Run: - -```bash -webcmd browser verify / --trace retain-on-failure -``` - -After the first passing run, read and save the fixture: - -```bash -webcmd site fixture get / --output /tmp/.json -``` - -Then tighten the saved fixture and write it back: - -- Add `notEmpty` for essential columns. -- Add `patterns` for URL, ID, date, or slug formats. -- Set realistic `rowCount`. -- Keep `types` narrow. - -```bash -webcmd site fixture put / /tmp/.json -``` - -Run verify again and confirm the fixture matches. diff --git a/skill-src/cli/webcmd-adapter-author/references/api-discovery.src.md b/skill-src/cli/webcmd-adapter-author/references/api-discovery.src.md deleted file mode 100644 index c896d904..00000000 --- a/skill-src/cli/webcmd-adapter-author/references/api-discovery.src.md +++ /dev/null @@ -1,255 +0,0 @@ -# API Discovery - -Use this after `site-recon.md` chooses Pattern A/B/C/D/E. The output of this file is a candidate endpoint plus evidence for the strategy note. - -Keep `--trace on --keep-tab true --window foreground` enabled while exploring browser-backed sites. - -## Section 0 - Preflight Red Lines - -Read these before endpoint verification. If you miss either one, you can spend the rest of discovery testing the wrong thing. - -### 0.1 Anti-bot and WAF gates decide whether Node fetch is valid - -Use `browser run` to inspect cookies and the response body manually. - -| Cookie or body signal | Vendor | Bare Node fetch or curl result | Strategy | -| --- | --- | --- | --- | -| `acw_sc__v2`, `acw_tc`, `ssxmod_itna`; body contains `arg1 = '32-HEX'` or `/ntc_captcha/` | Aliyun WAF | Slider HTML instead of real data | Verify the endpoint in browser context first; HTML-style cookie adapters can still end with Node-side fetch plus `page.getCookies()` | -| `__cf_bm`, `cf_clearance`, `__cfduid`; body contains `Cloudflare Ray ID` or `Checking your browser` | Cloudflare | TLS or browser fingerprint is rejected | Use a browser/session-aware probe first, then choose the adapter fetch route from `adapter-template.md` | -| `_abck`, `bm_sz`, `bm_sv` | Akamai | Often blocked even with cookies | Use a browser/session-aware probe first | -| Body contains `geetest` or `gt_captcha` | Geetest | Slider or puzzle challenge; no programmatic solution in this skill | Out of scope; stop or use a user-visible UI strategy | - -Rule: if any of these anti-bot or WAF signals appear, do not use bare Node fetch as endpoint verification. First prove the endpoint from the browser context or from a page on the target origin. After that, choose the final adapter strategy normally: JSON browser APIs may use `page.fetchJson()`, while HTML-style cookie adapters should keep using Node-side `fetch` with cookies read through `page.getCookies()`. - -### 0.2 Cross-subdomain fetch is CORS-blocked by default - -For example, a page on `jobs.51job.com` fetching an API on `cupid.51job.com` will usually hit a CORS preflight unless the API returns `Access-Control-Allow-Origin`. - -Probe it explicitly: - -```bash -webcmd --session browser run --stdin <<'JS' -await page.goto('https:///'); -return await page.evaluate(async () => { - try { - return await fetch('https:///api/...', { credentials: 'include' }).then(r => r.status); - } catch (error) { - return `cors:${error instanceof Error ? error.message : String(error)}`; - } -}); -JS -``` - -- A numeric status means CORS allows the request. -- `cors:...` or `TypeError: Failed to fetch` means the browser blocked it. - -When it is blocked, `credentials: include` is not a CORS fix across subdomains. It only asks the browser to send cookies; it does not grant cross-origin permission. Use this fallback order: - -1. Prefer a same-origin endpoint on the current subdomain. -2. Navigate to the target subdomain inside `browser run`, then fetch relative paths from that origin. -3. If the data is truly cross-origin and there is no same-origin alternative, use Section 5 intercept and capture the response from the page's own request. - -## Section 1 - Network Deep Read - -Use for Pattern A and for deeper data in Pattern B. - -```bash -webcmd --session browser run --stdin <<'JS' -const candidates = []; -page.on('response', async response => { - const url = response.url(); - const contentType = response.headers()['content-type'] || ''; - if (!url.includes('') && !/json|graphql/i.test(contentType)) return; - let sample = ''; - try { sample = (await response.text()).slice(0, 2000); } catch {} - candidates.push({ - url, - method: response.request().method(), - status: response.status(), - contentType, - sample, - }); -}); - -await page.goto(''); -await page.waitForLoadState('domcontentloaded'); -await page.waitForTimeout(1500); -return candidates.slice(0, 20); -JS -``` - -Inspect each candidate: - -- URL and method. -- Status code and content type. -- Query/body params. -- Request headers that appear auth-related. -- Response shape and whether it includes target data. -- Whether data is user-visible, not analytics, ads, experiments, or personalization noise. - -Reject candidates that only contain telemetry, unrelated recommendations, beacons, or layout metadata. - -Replay directly when possible: - -```bash -webcmd --session browser run --stdin <<'JS' -return await page.evaluate(async () => - fetch('', { credentials: 'include' }).then(r => r.text()) -); -JS -``` - -If Node-side replay works without page runtime state, prefer `PUBLIC_API` or `COOKIE_API`. If the endpoint only works in page context, document why before selecting `PAGE_FETCH`. - -For a request that exists only after a UI action, use `browser run` so the -listener is attached before the trigger: - -```js -const pending = page.waitForResponse( - response => response.url().includes('/api/target'), -); -await page.getByRole('button', { name: 'Load' }).click(); -const response = await pending; -return { - url: response.url(), - method: response.request().method(), - status: response.status(), - body: await response.json(), -}; -``` - -This is recon evidence only. Choose the adapter strategy from the verified -endpoint and UI evidence; do not copy the browser-run program into the adapter. - -## Section 2 - State Extraction - -Use for Pattern B. - -Look for: - -- `window.__INITIAL_STATE__` -- `window.__NEXT_DATA__` -- `window.__NUXT__` -- JSON in `