From e1e5f8574691a0fb5258075781f812802dac7d9c Mon Sep 17 00:00:00 2001 From: DavertMik Date: Mon, 14 Sep 2026 14:46:52 +0300 Subject: [PATCH 01/21] docs: design spec for mdq standalone package Extracts src/utils/markdown-query.ts into src/utils/mdq/ as a publish-ready package: MarkdownDoc + Selection, insert/remove verbs alongside query, a comment selector, frontmatter handling, JS-value matchers, and a planned jq-like CLI. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- .../specs/2026-09-14-mdq-package-design.md | 345 ++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 docs/superpowers/specs/2026-09-14-mdq-package-design.md diff --git a/docs/superpowers/specs/2026-09-14-mdq-package-design.md b/docs/superpowers/specs/2026-09-14-mdq-package-design.md new file mode 100644 index 00000000..a48a9012 --- /dev/null +++ b/docs/superpowers/specs/2026-09-14-mdq-package-design.md @@ -0,0 +1,345 @@ +# mdq — Markdown Query & Edit Package + +Date: 2026-09-14 +Status: Approved design, pending implementation + +## Goal + +Extract `src/utils/markdown-query.ts` into `src/utils/mdq/`, designed as a publishable +standalone package: query markdown *and* update it, with a jq-like CLI planned as a +second phase. + +The name `mdq` is unclaimed on npm (verified 404). Publishing is deferred; this change +makes the package publish-ready but adds no `package.json` or build script. + +## Constraints + +- **Zero explorbot imports.** `marked` is the only dependency. Frontmatter is parsed + in-package rather than pulling in `gray-matter`. +- **Two files**, per the module split below. +- 54 in-repo call sites must keep working; a re-export shim carries them. + +## Architecture + +``` +src/utils/mdq/ + query.ts selector grammar - token index - MarkdownDoc - Selection (reads) + edit.ts pure string -> string: splicing - whitespace - renderers - frontmatter + README.md public documentation +src/utils/markdown-query.ts re-export shim +tests/unit/mdq/*.test.ts +``` + +`edit.ts` exports only pure `string -> string` functions and never references a class. +`query.ts` owns both classes and delegates each write verb to exactly one `edit.ts` call. +This makes the split acyclic by construction. + +Two types: + +- **`MarkdownDoc`** — a whole document. Returned by `mdq()` and by every write. +- **`Selection`** — a set of matched ranges. Returned by `query()` and the sugar methods. + +One rule, stated in the README: **reads narrow, writes return the document.** + +## API + +### `mdq(source)` + +`mdq(source: string | MarkdownDoc): MarkdownDoc` + +Accepting a `MarkdownDoc` makes re-wrapping free. + +### `MarkdownDoc` + +| Method | Returns | Notes | +|---|---|---| +| `query(selector, matcher?)` | `Selection` | | +| `frontmatter()` | `Record` | `{}` when absent | +| `setFrontmatter(key, value)` | `MarkdownDoc` | `null` value deletes the key | +| `toString()` / `valueOf()` | `string` | full document, frontmatter included | + +Plus the shared sugar layer. + +### `Selection` — reads + +Narrow or extract; never mutate. + +| Method | Returns | Replaces | +|---|---|---| +| `query(selector, matcher?)` | `Selection` | sub-query, unchanged | +| `text()` | `string` | — (`get()` deprecated) | +| `count()` | `number` | | +| `exists()` | `boolean` | the `.count() > 0` idiom, 3 in-repo uses | +| `first()` / `last()` | `Selection` | | +| `at(n)` | `Selection` | new; sugar-path equivalent of DSL `[n]` | +| `slice(from?, to?)` | `Selection` | new; sugar-path equivalent of DSL `[a:b]` | +| `each()` | `Selection[]` | | +| `nodes()` | `NodeInfo[]` | `meta()` | +| `rows()` | `Record[]` | `toJson()` — it only ever handled tables | +| `entries()` | `Record` | `keyValue()` | +| `preceding()` / `following()` | `Selection` | `before()` / `after()` | + +`before`/`after` are renamed specifically to free those names from colliding with +`insertBefore`/`insertAfter`. + +### `Selection` — writes + +Every write returns `MarkdownDoc`, so edits chain in one expression. + +| Method | Signature | Notes | +|---|---|---| +| `replace(md)` | `(Markdown) => MarkdownDoc` | | +| `replaceEach(fn)` | `((Selection, number) => Markdown) => MarkdownDoc` | | +| `remove()` | `() => MarkdownDoc` | node **plus its adjacent `space` token** | +| `insertBefore(md)` | `(Markdown) => MarkdownDoc` | sibling | +| `insertAfter(md)` | `(Markdown) => MarkdownDoc` | sibling | +| `prepend(md)` | `(Markdown) => MarkdownDoc` | inside a section or list | +| `append(md)` | `(Markdown) => MarkdownDoc` | inside a section or list | +| `addRow(row)` | `(Record) => MarkdownDoc` | table only; re-aligns columns | +| `addItem(text)` | `(string) => MarkdownDoc` | list only; matches marker + indent | +| `setEntry(key, value)` | `(string, string \| null) => MarkdownDoc` | `null` deletes | + +Naming now pairs: `rows()`/`addRow()`, `entries()`/`setEntry()`, `nodes()`. + +Every verb that *takes* markdown accepts `Markdown = string | MarkdownDoc`, mirroring +`mdq()` itself. A `replaceEach` callback may therefore return a `MarkdownDoc` built by a +nested edit, without a `.toString()` hop. + +### Sugar layer + +Eleven methods on both classes: `section` `heading` `paragraph` `table` `list` `item` +`code` `blockquote` `comment` `html` `hr`. + +Each is `(matcher?, opts?) => Selection` and is *defined as* `query(sel, matcher)` — +documented as sugar, not a parallel implementation. Defined once on a shared base that +implements them in terms of an abstract `query()`, so the two classes do not duplicate it. + +Depth is an option rather than 12 near-duplicate methods: + +```js +mdq(src).section('API', { depth: 2 }) // DSL: query('section2("API")') +mdq(src).heading(/^f/i).at(0) // DSL: query('heading(/^f/i)[0]') +mdq(src).comment(/^test/) // DSL: query('comment(/^test/)') +``` + +### Exported types + +Declared at the end of their file, per repo convention. + +```ts +type Markdown = string | MarkdownDoc; +type Matcher = string | RegExp | ((text: string) => boolean); + +interface NodeInfo { + type: string; // 'heading' | 'paragraph' | 'table' | 'comment' | ... + depth: number | null; // heading level, else null + text: string; // unwrapped text; comment bodies without +} + +interface SelectorOptions { + depth?: 1 | 2 | 3 | 4 | 5 | 6; +} +``` + +`Matcher` semantics: + +- `string` — exact match (mirrors DSL `"x"`) +- `RegExp` — pattern, honoring its own flags +- function — predicate; needs no escaping at all + +This removes an existing wart. Today the repo hand-escapes to build selector strings: + +```js +const escaped = section.name.replace(/"/g, '\\"'); // researcher/focus.ts:77 +mdq(result.text).query(`section2(~"${escaped}")`); +``` + +### Deprecated aliases + +`get` `toJson` `keyValue` `setKeyValue` `meta` `before` `after` are kept, marked +`@deprecated`, and omitted from the README so the published surface reads clean. + +Aliases cover the read renames completely. They cannot shield the write return-type +change — see Migration. + +## Selector grammar + +Unchanged, plus one addition and three fixes. + +### `comment` (new) + +`html` tokens filtered to those that are comments. Not an alias for `html`: `
x
` +lexes as `html` too. + +- `comment` matches on the **inner** body, trimmed. `html` matches on raw. + This is required for anchored patterns — `/^test/` against `` only + works if the text is `test id=1`. +- Multi-line comments are a single token and keep their newlines in the matched text. +- **Inline comments are out of scope for 1.0.** `para with comment` lexes the + comment inside the paragraph token; it is not reachable as a block. Documented, not faked. + +`comment` and `html` together finish the `test-plan-markdown.ts` story: its hand-rolled +line parser (`src/utils/test-plan-markdown.ts:122+`) exists only because mdq could not +see `` and ``. + +### Fixes + +1. **Regex flags are honored.** Today flags are parsed then discarded + (`markdown-query.ts:90`) and `'i'` is hardcoded (`markdown-query.ts:156`), so `/x/` is + case-insensitive while `"x"` and `~"x"` are case-sensitive. After the fix `/x/i` is + insensitive and `/x/` is not. One production call site relies on the old behavior: + `researcher.ts:316` `section2(/^summary/)` becomes `/^summary/i`. Tests already write + flags explicitly. +2. **Unknown selectors throw.** Today `query('secton("A")')` silently matches nothing + (`markdown-query.ts:103-106`, `:356`). Unacceptable for a CLI. +3. **Table text-match widens to headers plus cells.** Today `getTokenText` returns headers + only (`markdown-query.ts:181`), so `table(~"GET")` can never match a cell. No call site + uses table text-matching, so this is safe. + +## Update semantics + +`marked` separators are uneven, and every write rule follows from this: + +| Token | `raw` | +|---|---| +| `heading` | `"# A\n\n"` — separators baked in | +| `paragraph` | `"para"` — no trailing newline | +| `space` | `"\n\n"` — a separate token | + +The token index therefore records each node's range **and its adjacent `space` range**. + +> **Invariant: mdq never leaves zero blank lines between blocks, and never more than one.** + +- `remove()` takes the node plus its trailing `space` — or its leading `space` when it is + the last block. Without this, removing a paragraph leaves a four-newline crater. This is + the most likely bug in the feature and gets dedicated tests. +- `insertAfter` / `append` normalize inserted markdown to one trailing `\n` and splice at + the boundary, never inside a space token. +- `append` on a section inserts before the next same-or-shallower heading, reusing the + existing `computeSections` end boundary. +- `addRow` re-renders the whole table so column pipes stay aligned. +- `addItem` copies the list's existing marker (`-`, `*`, `1.`) and indent. + +## Frontmatter + +Every `knowledge/` and `experience/` file opens with `---\nurl: /login\n---`, which +`marked` lexes as a setext h2 titled `url: /login`. + +mdq detects leading frontmatter, excludes it from the token index with offsets preserved +so edits splice correctly, and exposes it as data: + +```js +const doc = mdq(knowledgeFile); +doc.frontmatter(); // { url: '/login', wait: 1000 } +doc.query('h2').count(); // 0 — the --- block is not a heading +doc.setFrontmatter('wait', 2000).toString(); +``` + +## Errors + +`MdqError` base, with: + +- `MdqSelectorError` — malformed or unknown selector, carrying the offending index. +- `MdqOperationError` — a verb applied to the wrong node type, e.g. `addRow` on a paragraph. + +An **empty selection is a safe no-op**: reads return `''` / `[]`, writes return the +document unchanged. This preserves the existing "returns source unchanged when no matches" +test. + +## CLI (phase 2) + +The selector is the program, the file or stdin is the input, markdown is the default output. + +```bash +mdq '.h2' README.md # raw markdown of matches +cat plan.md | mdq 'section("API") table' -j # rows() as JSON +mdq 'comment(~"test")' plan.md --count +mdq 'section("FAQ")' doc.md --remove -i # edit in place +mdq 'table[0]' api.md --add-row '{"Method":"GET","Path":"/users"}' -i +``` + +Flags mirror library verbs exactly: `--remove` `--replace` `--insert-before` +`--insert-after` `--prepend` `--append` `--add-row` `--add-item` `--set k=v`, plus +`-i/--in-place`, `-j/--json`, `-c/--count`, `-t/--text`, `--frontmatter`. + +Built with Commander, per repo convention. Exit codes compose like grep: **0** match, +**1** no match, **2** usage or selector error. + +1.0 reads one file or stdin. Multi-file input is out of scope. + +## Testing + +Port the existing 801-line suite first — it is the regression net for all 54 call sites. +Then add coverage for what is new or newly specified: + +- whitespace craters on `remove` (the invariant above) +- chained multi-edits through `MarkdownDoc` +- `addRow` column alignment; `addItem` marker and indent matching +- frontmatter round-trip, including a file whose body has its own `---` +- `comment` inner-text matching, multi-line comments, and `html` versus `comment` +- `MdqSelectorError` on unknown selectors and malformed input +- sugar equivalence: every sugar call equals its `query()` form + +## Migration + +Two steps. Only the second carries risk. + +1. `src/utils/markdown-query.ts` becomes a re-export shim. **All 54 call sites keep + working untouched.** +2. A sweep updates imports, then fixes the call sites the return-type change breaks. + +Deprecated read aliases mean **no read call site changes**. Writes are not shielded: a +verb that returned `string` now returns `MarkdownDoc`. That breaks three classes of site, +nine in total. + +**(a) Assignment into a `string`-typed target** — 7 sites, each needs `.toString()`: + +| Site | Target | +|---|---| +| `experience-tracker.ts:265` | `content` (inferred `string`) | +| `experience-tracker.ts:289` | `combined` (inferred `string`) | +| `researcher/deep-analysis.ts:129` | `let updated: string` | +| `researcher/locators.ts:307` | `result.text` | +| `researcher/locators.ts:309` | `result.text` | +| `researcher/pagination.ts:61` | `result.text` | +| `researcher/research-result.ts:57` | `section.rawMarkdown` | + +**(b) A string method called on the result** — 2 sites: + +- `planner.ts:304` — `.replace('').trim()` +- `planner.ts:322` — `const kept = ...replace('')`, then `kept.trimEnd()` on line 324 + +**(c) Returned from a `replaceEach` callback** — `deep-analysis.ts:542`. **Resolved by +design**, not by migration: callbacks accept `Markdown`, so returning a `MarkdownDoc` is +valid. No edit needed. + +Sites that flow the result straight back into `mdq()` — `planner.ts:303`, `planner.ts:405` +— keep working unchanged, because `mdq()` accepts a `MarkdownDoc`. + +`tsc` is the complete detector for this class of break: every one of the nine surfaces as +a type error. The migration step is therefore *run `tsc` over the changed files and fix +what it reports*, with the table above as the expected result rather than the whole story. + +Also in the sweep: `researcher.ts:316` gains its `i` flag (`section2(/^summary/i)`). + +### Hazards + +- **`.claude/worktrees/**` holds four stale copies** of `markdown-query.ts` and its + consumers. Every grep or sed sweep must exclude that path. +- **`tsc` runs with `--noCheck` in CI.** The nine breaks above are *exactly* the errors a + type-check would raise, and CI raises none of them — a fully green build proves nothing + here. Run `tsc` manually over the changed files before considering the sweep done. + +## Style compliance + +The current file violates several repo rules that the rewrite fixes: types belong at the +end of the file, ternaries are banned (`markdown-query.ts:30`, `:34`, `:129-130`, `:295`), +and the `switch` in `matchText` should be early returns. + +## Deliberately out of scope + +- Publishing: no `package.json`, no build script, no npm release in this change. +- Row-level and item-level *selectors* (`addRow` has no `removeRow` partner). A future + `row(...)` selector is the right shape for that; guessing at it now is premature. +- Inline HTML comments. +- Multi-file CLI input. From ad7c8f41bcef8155fa97d67a800353e2a3afa9bf Mon Sep 17 00:00:00 2001 From: DavertMik Date: Mon, 14 Sep 2026 18:44:15 +0300 Subject: [PATCH 02/21] docs: resolve open questions in mdq spec Frontmatter uses yaml's Document API (comment-preserving) rather than a hand-rolled parser; MarkdownDoc gains append/prepend; leading '.' is accepted in the CLI grammar; documents a fourth migration breakage class where MarkdownDoc === string silently stops a guard from firing. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- .../specs/2026-09-14-mdq-package-design.md | 82 +++++++++++++++---- 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/docs/superpowers/specs/2026-09-14-mdq-package-design.md b/docs/superpowers/specs/2026-09-14-mdq-package-design.md index a48a9012..3edc2381 100644 --- a/docs/superpowers/specs/2026-09-14-mdq-package-design.md +++ b/docs/superpowers/specs/2026-09-14-mdq-package-design.md @@ -14,8 +14,8 @@ makes the package publish-ready but adds no `package.json` or build script. ## Constraints -- **Zero explorbot imports.** `marked` is the only dependency. Frontmatter is parsed - in-package rather than pulling in `gray-matter`. +- **Zero explorbot imports.** Two dependencies only: `marked` for markdown, `yaml` for + frontmatter. Both are already repo deps (`marked` ^16.2.0, `yaml` ^2.8.3). - **Two files**, per the module split below. - 54 in-repo call sites must keep working; a re-export shim carries them. @@ -24,15 +24,16 @@ makes the package publish-ready but adds no `package.json` or build script. ``` src/utils/mdq/ query.ts selector grammar - token index - MarkdownDoc - Selection (reads) - edit.ts pure string -> string: splicing - whitespace - renderers - frontmatter + edit.ts pure edits over (source, ranges): splicing - whitespace - renderers README.md public documentation src/utils/markdown-query.ts re-export shim tests/unit/mdq/*.test.ts ``` -`edit.ts` exports only pure `string -> string` functions and never references a class. -`query.ts` owns both classes and delegates each write verb to exactly one `edit.ts` call. -This makes the split acyclic by construction. +`edit.ts` exports pure functions taking source text plus ranges or tokens, and returning +new source text. It imports types from `query.ts` type-only and never references a class +value. `query.ts` owns both classes and delegates each write verb to exactly one `edit.ts` +call. This keeps the split acyclic by construction. Two types: @@ -56,10 +57,17 @@ Accepting a `MarkdownDoc` makes re-wrapping free. | `query(selector, matcher?)` | `Selection` | | | `frontmatter()` | `Record` | `{}` when absent | | `setFrontmatter(key, value)` | `MarkdownDoc` | `null` value deletes the key | +| `append(md)` | `MarkdownDoc` | add a block at end of document | +| `prepend(md)` | `MarkdownDoc` | add a block at start of body, after frontmatter | | `toString()` / `valueOf()` | `string` | full document, frontmatter included | Plus the shared sugar layer. +`append`/`prepend` exist because "add a section to the end of the document" otherwise has +no clean path — only the `section().last().insertAfter(...)` workaround. There is a real +call site: `deep-analysis.ts:131` builds it by hand today as +`` `${cached.trimEnd()}\n\n# Extended Research\n\n...` ``. + ### `Selection` — reads Narrow or extract; never mutate. @@ -93,8 +101,8 @@ Every write returns `MarkdownDoc`, so edits chain in one expression. | `remove()` | `() => MarkdownDoc` | node **plus its adjacent `space` token** | | `insertBefore(md)` | `(Markdown) => MarkdownDoc` | sibling | | `insertAfter(md)` | `(Markdown) => MarkdownDoc` | sibling | -| `prepend(md)` | `(Markdown) => MarkdownDoc` | inside a section or list | -| `append(md)` | `(Markdown) => MarkdownDoc` | inside a section or list | +| `prepend(md)` | `(Markdown) => MarkdownDoc` | inside a section or list; `MdqOperationError` on a leaf node | +| `append(md)` | `(Markdown) => MarkdownDoc` | inside a section or list; `MdqOperationError` on a leaf node | | `addRow(row)` | `(Record) => MarkdownDoc` | table only; re-aligns columns | | `addItem(text)` | `(string) => MarkdownDoc` | list only; matches marker + indent | | `setEntry(key, value)` | `(string, string \| null) => MarkdownDoc` | `null` deletes | @@ -147,6 +155,12 @@ interface SelectorOptions { - `RegExp` — pattern, honoring its own flags - function — predicate; needs no escaping at all +Note the consequence for `comment`: a `string` matcher is **exact**, and this repo's own +test-plan comments are multi-line (``). So +`comment('test')` matches only a bare ``; reaching the multi-line ones needs +`comment(/^test/)` or a predicate. Exactness is the consistent rule and is kept, but it is +the one place the sugar is likely to surprise. + This removes an existing wart. Today the repo hand-escapes to build selector strings: ```js @@ -226,11 +240,30 @@ Every `knowledge/` and `experience/` file opens with `---\nurl: /login\n---`, wh `marked` lexes as a setext h2 titled `url: /login`. mdq detects leading frontmatter, excludes it from the token index with offsets preserved -so edits splice correctly, and exposes it as data: +so edits splice correctly, and exposes it as data. + +Reading and writing both go through `yaml`'s **Document API** (`YAML.parseDocument`), not +`parse`/`stringify`. That buys two things a hand-rolled parser cannot: correctness on +nested maps, lists and block scalars — the Jekyll/Astro/Obsidian files that justify the +feature — and **comment preservation through a write**, verified: + +```yaml +# a leading comment <- survives setFrontmatter('wait', 2000) +url: /login +wait: 2000 +tags: + - auth + - smoke +nested: + key: value # trailing note <- also survives +``` + +`gray-matter` is deliberately not used: `knowledge-tracker.ts` keeps it for its own +purposes, but a published package should not carry it to do what `yaml` already does. ```js const doc = mdq(knowledgeFile); -doc.frontmatter(); // { url: '/login', wait: 1000 } +doc.frontmatter(); // { url: '/login', wait: 1000, tags: ['auth'] } doc.query('h2').count(); // 0 — the --- block is not a heading doc.setFrontmatter('wait', 2000).toString(); ``` @@ -250,8 +283,12 @@ test. The selector is the program, the file or stdin is the input, markdown is the default output. +A leading `.` is accepted and ignored, so muscle memory from jq (`mdq '.h2'`) works. It is +sugar in the grammar, not a separate syntax — without it the new "unknown selectors throw" +rule would reject the most natural thing a jq user types first. + ```bash -mdq '.h2' README.md # raw markdown of matches +mdq 'h2' README.md # raw markdown of matches cat plan.md | mdq 'section("API") table' -j # rows() as JSON mdq 'comment(~"test")' plan.md --count mdq 'section("FAQ")' doc.md --remove -i # edit in place @@ -289,8 +326,8 @@ Two steps. Only the second carries risk. 2. A sweep updates imports, then fixes the call sites the return-type change breaks. Deprecated read aliases mean **no read call site changes**. Writes are not shielded: a -verb that returned `string` now returns `MarkdownDoc`. That breaks three classes of site, -nine in total. +verb that returned `string` now returns `MarkdownDoc`. That breaks four classes of site, +at least eleven in total. **(a) Assignment into a `string`-typed target** — 7 sites, each needs `.toString()`: @@ -303,6 +340,7 @@ nine in total. | `researcher/locators.ts:309` | `result.text` | | `researcher/pagination.ts:61` | `result.text` | | `researcher/research-result.ts:57` | `section.rawMarkdown` | +| `researcher/research-result.ts:58` | `this.text` | **(b) A string method called on the result** — 2 sites: @@ -313,11 +351,23 @@ nine in total. design**, not by migration: callbacks accept `Markdown`, so returning a `MarkdownDoc` is valid. No edit needed. +**(d) Compared against a string — the dangerous one.** `research-result.ts:56`: + +```js +const updated = sectionQuery.query('table').replace(`${newTable.trimEnd()}\n`); +if (updated === this.text) return; // MarkdownDoc === string is always false +``` + +This does not crash. The guard silently stops firing and the method starts doing work it +used to skip. `tsc` does flag it — comparing types with no overlap is an error — which is +precisely why the manual type-check below is not optional. Every `replace`/`setEntry` +result used in an equality or truthiness test must be audited, not just the ones that fail +to compile. + Sites that flow the result straight back into `mdq()` — `planner.ts:303`, `planner.ts:405` — keep working unchanged, because `mdq()` accepts a `MarkdownDoc`. -`tsc` is the complete detector for this class of break: every one of the nine surfaces as -a type error. The migration step is therefore *run `tsc` over the changed files and fix +`tsc` is the complete detector for these breaks: every one surfaces as a type error. The migration step is therefore *run `tsc` over the changed files and fix what it reports*, with the table above as the expected result rather than the whole story. Also in the sweep: `researcher.ts:316` gains its `i` flag (`section2(/^summary/i)`). @@ -339,6 +389,8 @@ and the `switch` in `matchText` should be early returns. ## Deliberately out of scope - Publishing: no `package.json`, no build script, no npm release in this change. +- Frontmatter formats other than YAML (TOML `+++`, JSON) — detected and skipped from the + token index, but not parsed. - Row-level and item-level *selectors* (`addRow` has no `removeRow` partner). A future `row(...)` selector is the right shape for that; guessing at it now is premature. - Inline HTML comments. From a0dc4a780d2ff4744159730fe661c0ec3346f50a Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:08:48 +0300 Subject: [PATCH 03/21] docs: implementation plan for mdq package 11 tasks, 68 steps. Ordering keeps the repo green at every boundary: port behind a shim first, migrate the 11 write-return-type breaks second, then add features additively. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- .../plans/2026-09-15-mdq-package.md | 2029 +++++++++++++++++ 1 file changed, 2029 insertions(+) create mode 100644 docs/superpowers/plans/2026-09-15-mdq-package.md diff --git a/docs/superpowers/plans/2026-09-15-mdq-package.md b/docs/superpowers/plans/2026-09-15-mdq-package.md new file mode 100644 index 00000000..ecedbe44 --- /dev/null +++ b/docs/superpowers/plans/2026-09-15-mdq-package.md @@ -0,0 +1,2029 @@ +# mdq Package Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Extract `src/utils/markdown-query.ts` into `src/utils/mdq/` as a publish-ready package that can both query and update markdown, then add a jq-like CLI. + +**Architecture:** Two classes in two files. `query.ts` owns the selector grammar, the token index, `MarkdownDoc` and `Selection`. `edit.ts` holds pure functions that take source text plus ranges and return new source text; it imports types from `query.ts` type-only and never touches a class value, which keeps the split acyclic. Reads narrow to a `Selection`; writes return a `MarkdownDoc`, so edits chain. + +**Tech Stack:** TypeScript, Bun, `marked` ^16.2.0 (markdown), `yaml` ^2.8.3 (frontmatter), `vitest` API via `bun test`, Commander (CLI only). + +**Spec:** `docs/superpowers/specs/2026-09-14-mdq-package-design.md` + +## Global Constraints + +- **Two dependencies only.** `marked` and `yaml`. No imports from anywhere else in explorbot — not `src/utils/`, not `src/commands/`, nothing. This is what makes the package extractable. +- **Repo style rules** (from `CLAUDE.md`, all enforced in review): + - No comments unless explicitly requested. + - No ternaries. No `...(cond ? {k:v} : {})` spread. + - Prefer early return over `if/else`. + - Types and interfaces at the **end** of the file. + - Private methods after public methods. + - Use `?.` rather than chained `&&`. +- `bun run format` after each code change; `bun run lint:fix` after each task. +- **Never run the regression workflow** and never add the `regression` label. +- Bun only. Never Node. + +## Baseline facts (measured 2026-09-15, do not re-derive) + +- `bun test tests/unit/markdown-query.test.ts` → **110 pass, 0 fail**. This suite is the regression net for all 54 call sites; it must stay green at every task boundary. +- `bunx tsc -p tsconfig.json --noEmit` → **1004 errors repo-wide**. A clean `tsc` is not achievable and is not the goal. Only the scoped check in Task 3 matters. +- Of those 1004, the files this plan touches own exactly **2**, both pre-existing and unrelated to mdq: + ``` + src/ai/researcher/locators.ts(247,41): error TS2339: Property 'playwrightLocatorCount' does not exist on type 'Explorer'. + src/ai/researcher/locators.ts(247,65): error TS7006: Parameter 'page' implicitly has an 'any' type. + ``` + These two are the expected output of the Task 3 scoped check. **Three or more means the migration is incomplete.** +- CI runs `tsc` with `--noCheck`. A green CI proves nothing about types here. + +## File Structure + +| File | Responsibility | +|---|---| +| `src/utils/mdq/query.ts` | Selector grammar, token index (frontmatter-aware), `MarkdownDoc`, `Selection`, sugar layer, error classes | +| `src/utils/mdq/edit.ts` | Pure edits over `(source, ranges)`: splicing, whitespace normalization, table/list renderers, entry and frontmatter rewriting | +| `src/utils/mdq/cli.ts` | CLI argument handling and output formatting (Task 10) | +| `src/utils/mdq/README.md` | Public documentation (Task 9) | +| `src/utils/markdown-query.ts` | Re-export shim so all 54 existing call sites keep working | +| `bin/mdq.ts` | Thin CLI entry delegating to `cli.ts` (Task 10) | +| `tests/unit/mdq/*.test.ts` | Test suites, one per concern | + +--- + +### Task 1: Scaffold the package with a frontmatter-aware token index + +Move the parser to its new home and teach it the one thing it gets wrong today: a leading `---` block is frontmatter, not a setext heading. + +**Files:** +- Create: `src/utils/mdq/query.ts` +- Create: `src/utils/mdq/edit.ts` +- Create: `tests/unit/mdq/frontmatter.test.ts` + +**Interfaces:** +- Consumes: nothing (first task) +- Produces: + - `buildTokenIndex(source: string): MatchedRange[]` — ranges are absolute offsets into `source`, frontmatter excluded + - `splitFrontmatter(source: string): { raw: string; body: string; offset: number }` + - `interface MatchedRange { token: Token; start: number; length: number; trailing?: { start: number; length: number }; innerTokens?: MatchedRange[] }` + +The `trailing` field is new and load-bearing: `marked` emits `space` tokens as siblings (a `paragraph` raw is `"para"` with no newline, followed by a separate `space` raw of `"\n\n"`), so every write verb needs to know where a node's separator lives. + +- [ ] **Step 1: Write the failing test** + +```ts +// tests/unit/mdq/frontmatter.test.ts +import { describe, expect, it } from 'vitest'; +import { splitFrontmatter } from '../../../src/utils/mdq/edit.ts'; +import { buildTokenIndex } from '../../../src/utils/mdq/query.ts'; + +describe('splitFrontmatter', () => { + it('splits a leading yaml block from the body', () => { + const src = '---\nurl: /login\nwait: 1000\n---\n\n# Title\n'; + const fm = splitFrontmatter(src); + expect(fm.raw).toBe('url: /login\nwait: 1000'); + expect(fm.body).toBe('\n# Title\n'); + expect(fm.offset).toBe(src.length - fm.body.length); + }); + + it('returns no frontmatter when the document does not open with ---', () => { + const fm = splitFrontmatter('# Title\n\n---\n'); + expect(fm.raw).toBe(''); + expect(fm.offset).toBe(0); + }); + + it('treats an unterminated --- as body, not frontmatter', () => { + const fm = splitFrontmatter('---\nnot closed\n'); + expect(fm.raw).toBe(''); + expect(fm.offset).toBe(0); + }); +}); + +describe('buildTokenIndex', () => { + it('excludes frontmatter so it is never lexed as a setext heading', () => { + const ranges = buildTokenIndex('---\nurl: /login\n---\n\n# Title\n'); + expect(ranges.filter((r) => r.token.type === 'heading')).toHaveLength(1); + expect(ranges.every((r) => r.start >= 20)).toBe(true); + }); + + it('keeps offsets absolute so slicing the original source works', () => { + const src = '---\nurl: /x\n---\n\n# Title\n'; + const ranges = buildTokenIndex(src); + const heading = ranges.find((r) => r.token.type === 'heading'); + expect(src.slice(heading.start, heading.start + heading.length)).toBe('# Title\n'); + }); + + it('records the trailing space token of a paragraph', () => { + const src = 'para\n\n# Next\n'; + const ranges = buildTokenIndex(src); + const para = ranges.find((r) => r.token.type === 'paragraph'); + expect(para.length).toBe(4); + expect(para.trailing).toEqual({ start: 4, length: 2 }); + }); + + it('leaves trailing undefined for a node with no following space', () => { + const ranges = buildTokenIndex('# Only\n'); + expect(ranges[0].trailing).toBeUndefined(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test tests/unit/mdq/frontmatter.test.ts` +Expected: FAIL — cannot resolve `src/utils/mdq/query.ts` + +- [ ] **Step 3: Create query.ts with the index** + +Copy `src/utils/markdown-query.ts` to `src/utils/mdq/query.ts` verbatim first, then apply these three changes. + +Create `src/utils/mdq/edit.ts` holding the frontmatter grammar. It lives here from the +start so that later tasks add to this file rather than moving code between the two — +`query.ts` imports the value, `edit.ts` only ever imports types back, so the split stays +acyclic: + +```ts +export function splitFrontmatter(source: string): FrontmatterSplit { + if (!source.startsWith('---')) return { raw: '', body: source, offset: 0 }; + const match = source.match(/^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/); + if (!match) return { raw: '', body: source, offset: 0 }; + return { raw: match[1], body: source.slice(match[0].length), offset: match[0].length }; +} +``` + +Replace the body of `buildTokenIndex`: + +```ts +export function buildTokenIndex(source: string): MatchedRange[] { + const { body, offset } = splitFrontmatter(source); + const tokens = marked.lexer(body); + const ranges: MatchedRange[] = []; + let cursor = offset; + + for (const token of tokens) { + const raw = (token as any).raw || ''; + if (token.type === 'space') { + const previous = ranges[ranges.length - 1]; + if (previous) previous.trailing = { start: cursor, length: raw.length }; + cursor += raw.length; + continue; + } + ranges.push({ token, start: cursor, length: raw.length }); + cursor += raw.length; + } + + return ranges; +} +``` + +Note this also stops `space` tokens from appearing as matchable ranges, which they never should have been. + +`query.ts` imports it and re-exports for convenience: + +```ts +import { splitFrontmatter } from './edit.ts'; + +export { splitFrontmatter }; +``` + +Add to the type block at the **end** of `edit.ts`: + +```ts +export interface FrontmatterSplit { + raw: string; + body: string; + offset: number; +} +``` + +and extend the existing `MatchedRange` at the end of `query.ts`: + +```ts +export interface MatchedRange { + token: Token; + start: number; + length: number; + trailing?: { start: number; length: number }; + innerTokens?: MatchedRange[]; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test tests/unit/mdq/frontmatter.test.ts` +Expected: PASS (8 tests) + +- [ ] **Step 5: Format, lint and commit** + +```bash +bun run format && bun run lint:fix +git add src/utils/mdq/query.ts src/utils/mdq/edit.ts tests/unit/mdq/frontmatter.test.ts +git commit -m "feat(mdq): frontmatter-aware token index" +``` + +--- + +### Task 2: Port the public API behind a shim, behaviour unchanged + +Get every existing call site running against the new file with **zero behaviour change**. Writes still return `string` here — flipping them is Task 3. This task is the safety net for everything after it. + +**Files:** +- Modify: `src/utils/mdq/query.ts` +- Modify: `src/utils/markdown-query.ts` (becomes a shim) +- Create: `tests/unit/mdq/query.test.ts` (moved from `tests/unit/markdown-query.test.ts`) +- Delete: `tests/unit/markdown-query.test.ts` + +**Interfaces:** +- Consumes: `buildTokenIndex`, `splitFrontmatter`, `MatchedRange` from Task 1 +- Produces: `mdq(source: string): MarkdownQuery`, class `MarkdownQuery`, `parseQuery`, all existing methods unchanged + +- [ ] **Step 1: Move the test file and repoint its import** + +```bash +git mv tests/unit/markdown-query.test.ts tests/unit/mdq/query.test.ts +``` + +Change line 2 of the moved file from: + +```ts +import { mdq, parseQuery } from '../../src/utils/markdown-query.ts'; +``` + +to: + +```ts +import { mdq, parseQuery } from '../../../src/utils/mdq/query.ts'; +``` + +- [ ] **Step 2: Run the suite to verify it fails** + +Run: `bun test tests/unit/mdq/query.test.ts` +Expected: FAIL — `mdq` / `parseQuery` are not yet exported from `query.ts`, or section tests fail because `expandSectionRanges` still assumes `space` tokens are present + +- [ ] **Step 3: Restore the full API in query.ts** + +Everything from the original `markdown-query.ts` below `buildTokenIndex` — `matchText`, `entryKey`, `getTokenText`, `getHeadingDepth`, `isSectionSelector`, `getSectionDepth`, `selectorToTokenType`, `computeSections`, `extractListItems`, `applyIndexSlice`, `expandSectionRanges`, `executeSegments`, `class MarkdownQuery`, `mdq` — carries over unchanged, except: + +`computeSections` must extend a section's range to include the last inner node's trailing space, since `space` tokens are no longer separate ranges: + +```ts +for (let j = i + 1; j < candidates.length; j++) { + const nextRange = candidates[j]; + if (nextRange.token.type === 'heading' && (nextRange.token as Tokens.Heading).depth <= depth) break; + innerTokens.push(nextRange); + endOffset = nextRange.start + nextRange.length; + if (nextRange.trailing) endOffset = nextRange.trailing.start + nextRange.trailing.length; +} +``` + +Move every `export interface` / `export type` to the end of the file, and replace the ternaries at the original lines 30, 34, 129-130 and 295 with early returns. + +- [ ] **Step 4: Run the suite to verify it passes** + +Run: `bun test tests/unit/mdq/query.test.ts` +Expected: PASS — **110 tests**, the same count as before the move + +- [ ] **Step 5: Replace markdown-query.ts with a shim** + +```ts +export * from './mdq/query.ts'; +``` + +- [ ] **Step 6: Verify every existing call site still works** + +Run: `bun test tests/unit/` +Expected: PASS, no new failures versus the pre-task run + +- [ ] **Step 7: Format, lint and commit** + +```bash +bun run format && bun run lint:fix +git add -A src/utils tests/unit +git commit -m "refactor(mdq): move markdown-query into src/utils/mdq behind a shim" +``` + +--- + +### Task 3: Flip write verbs to return MarkdownDoc and migrate every call site + +The one risky task. It ends with the repo green and every break fixed. + +**Files:** +- Modify: `src/utils/mdq/query.ts` +- Modify: `src/experience-tracker.ts:265`, `:289` +- Modify: `src/ai/planner.ts:304`, `:322` +- Modify: `src/ai/researcher/deep-analysis.ts:129` +- Modify: `src/ai/researcher/locators.ts:307`, `:309` +- Modify: `src/ai/researcher/pagination.ts:61` +- Modify: `src/ai/researcher/research-result.ts:56`, `:57`, `:58` +- Modify: `src/ai/researcher.ts:316` +- Modify: `tests/unit/mdq/query.test.ts` + +**Interfaces:** +- Consumes: `MarkdownQuery`, `mdq` from Task 2 +- Produces: + - `class MarkdownDoc` — `query()`, `toString()`, `valueOf()` + - `class Selection` — all reads, plus writes returning `MarkdownDoc` + - `type Markdown = string | MarkdownDoc` + - `mdq(source: Markdown): MarkdownDoc` + - Deprecated alias `MarkdownQuery = Selection` + +- [ ] **Step 1: Write the failing test** + +```ts +// append to tests/unit/mdq/query.test.ts +describe('MarkdownDoc chaining', () => { + const md = '# T\n\n## A\n\npara\n\n## B\n\nother\n'; + + it('returns a MarkdownDoc from a write so edits chain', () => { + const out = mdq(md).query('h2("A")').replace('## Z\n\n').query('h2').count(); + expect(out).toBe(2); + }); + + it('stringifies to the full document', () => { + expect(mdq(md).query('h2("A")').replace('## Z\n\n').toString()).toContain('## Z'); + }); + + it('accepts a MarkdownDoc as a source', () => { + const doc = mdq(md).query('h2("A")').replace('## Z\n\n'); + expect(mdq(doc).query('h2').count()).toBe(2); + }); + + it('accepts a MarkdownDoc returned from a replaceEach callback', () => { + const out = mdq(md) + .query('h2') + .replaceEach((section) => mdq(section.text()).query('h2').replace('### x\n\n')) + .toString(); + expect(out).toContain('### x'); + expect(out).not.toContain('## A'); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test tests/unit/mdq/query.test.ts -t 'MarkdownDoc chaining'` +Expected: FAIL — `.query is not a function` on the string returned by `replace` + +- [ ] **Step 3: Split MarkdownQuery into MarkdownDoc and Selection** + +`MarkdownDoc` holds the source. `Selection` holds source plus matches. Every write on `Selection` ends by wrapping its result: + +```ts +export class MarkdownDoc { + private source: string; + + constructor(source: string) { + this.source = source; + } + + query(selector: string): Selection { + const segments = parseQuery(selector); + const candidates = expandSectionRanges(buildTokenIndex(this.source)); + return new Selection(this.source, executeSegments(candidates, segments)); + } + + toString(): string { + return this.source; + } + + valueOf(): string { + return this.source; + } +} +``` + +In `Selection`, each write returns `new MarkdownDoc(...)` instead of a raw string. `replaceEach` accepts `Markdown` back from its callback: + +```ts + replaceEach(replacer: (match: Selection, index: number) => Markdown): MarkdownDoc { + // ... unchanged range logic ... + const replacements = kept.map((range, index) => String(replacer(new Selection(this.source, [range]), index))); + // ... unchanged splice loop ... + return new MarkdownDoc(result); + } +``` + +`mdq` accepts either: + +```ts +export function mdq(source: Markdown): MarkdownDoc { + return new MarkdownDoc(String(source)); +} +``` + +At the end of the file: + +```ts +export type Markdown = string | MarkdownDoc; + +/** @deprecated Use Selection. */ +export const MarkdownQuery = Selection; +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test tests/unit/mdq/query.test.ts -t 'MarkdownDoc chaining'` +Expected: PASS (4 tests) + +- [ ] **Step 5: Fix the write assertions in the ported suite** + +The existing `replace` tests assert against a string. Wrap each in `String(...)`, for example: + +```ts + it('should replace matched content', () => { + const result = String(mdq(sampleMarkdown).query('heading("FAQ")').replace('## Questions\n')); + expect(result).toContain('## Questions'); + expect(result).not.toContain('## FAQ'); + }); +``` + +Apply the same to every assertion in the `replace`, `setKeyValue` and `edge cases` describes that compares a write result to a string. + +Run: `bun test tests/unit/mdq/query.test.ts` +Expected: PASS — 114 tests + +- [ ] **Step 6: Migrate the four breakage classes** + +**(a) Assignment into a `string`-typed target** — append `.toString()`: + +| File:line | Change | +|---|---| +| `src/experience-tracker.ts:265` | `content = sections[sections.length - 1].replace('').toString();` | +| `src/experience-tracker.ts:289` | `combined = mdq(combined).query('code').replace('').toString();` | +| `src/ai/researcher/deep-analysis.ts:129` | `updated = extQuery.replace(\`${existing}\n\n${sectionMarkdown}\n\`).toString();` | +| `src/ai/researcher/locators.ts:307` | `result.text = sectionQuery.query('blockquote[0]').setKeyValue('Container', \`'${newCss}'\`).toString();` | +| `src/ai/researcher/locators.ts:309` | `result.text = sectionQuery.query('blockquote[0]').replace('').toString();` | +| `src/ai/researcher/pagination.ts:61` | `result.text = sectionQuery.query('blockquote[0]').setKeyValue('Pagination', strategy).toString();` | +| `src/ai/researcher/research-result.ts:57` | `section.rawMarkdown = mdq(section.rawMarkdown).query('table').replace(\`${newTable.trimEnd()}\n\`).toString();` | + +**(b) A string method called on the result:** + +`src/ai/planner.ts:304`: +```ts + const body = mdq(withoutHeadings).query('hr').replace('').toString().trim(); +``` + +`src/ai/planner.ts:322`: +```ts + const kept = mdq(section.text()).query('blockquote[10:]').replace('').toString(); +``` + +**(c) Returned from a `replaceEach` callback** — `deep-analysis.ts:542` needs **no change**. The callback signature accepts `Markdown`. + +**(d) Compared against a string — read this one carefully.** `research-result.ts:55-58` currently reads: + +```ts + const updated = sectionQuery.query('table').replace(`${newTable.trimEnd()}\n`); + if (updated === this.text) return; + section.rawMarkdown = mdq(section.rawMarkdown).query('table').replace(`${newTable.trimEnd()}\n`); + this.text = updated; +``` + +`updated` is now a `MarkdownDoc`, so `updated === this.text` is **always false** — the guard silently stops firing and the method starts doing work it used to skip. Convert once, at the top: + +```ts + const updated = sectionQuery.query('table').replace(`${newTable.trimEnd()}\n`).toString(); + if (updated === this.text) return; + section.rawMarkdown = mdq(section.rawMarkdown).query('table').replace(`${newTable.trimEnd()}\n`).toString(); + this.text = updated; +``` + +**Leave alone** — these already work because `mdq()` accepts a `MarkdownDoc`: `planner.ts:303`, `planner.ts:405`. + +- [ ] **Step 7: Fix the one regex call site** + +`src/ai/researcher.ts:316` relies on regex matching being implicitly case-insensitive. Task 4 removes that. Make the flag explicit now so the two changes never overlap: + +```ts + const summaryText = mdq(result.text).query('section2(/^summary/i)').query('paragraph[0]').text().trim(); +``` + +- [ ] **Step 8: Verify with the scoped type check** + +```bash +bunx tsc -p tsconfig.json --noEmit 2>&1 | grep -E "^(src/utils/mdq/|src/utils/markdown-query|src/experience-tracker|src/ai/planner|src/ai/researcher)" +``` + +Expected: **exactly these two lines and nothing else.** + +``` +src/ai/researcher/locators.ts(247,41): error TS2339: Property 'playwrightLocatorCount' does not exist on type 'Explorer'. +src/ai/researcher/locators.ts(247,65): error TS7006: Parameter 'page' implicitly has an 'any' type. +``` + +Any third line is an unmigrated call site. Fix it before continuing — CI will not catch it, because CI runs `tsc --noCheck`. + +- [ ] **Step 9: Run the full unit suite** + +Run: `bun test tests/unit/` +Expected: PASS, no new failures + +- [ ] **Step 10: Format, lint and commit** + +```bash +bun run format && bun run lint:fix +git add -A src tests +git commit -m "feat(mdq): writes return MarkdownDoc so edits chain" +``` + +--- + +### Task 4: Selector additions — comment, html, honest regex flags, loud failures + +Four grammar changes, all additive now that Task 3 pre-fixed the one regex call site. + +**Files:** +- Modify: `src/utils/mdq/query.ts` +- Create: `tests/unit/mdq/selectors.test.ts` + +**Interfaces:** +- Consumes: `parseQuery`, `getTokenText`, `selectorToTokenType`, `matchText` from Task 2 +- Produces: `class MdqError extends Error`, `class MdqSelectorError extends MdqError` (with `index: number`), selectors `comment` and `html` + +- [ ] **Step 1: Write the failing test** + +```ts +// tests/unit/mdq/selectors.test.ts +import { describe, expect, it } from 'vitest'; +import { MdqSelectorError, mdq } from '../../../src/utils/mdq/query.ts'; + +const doc = ` + +## Plan + + + +para with comment + +
a block
+ +| Method | Path | +|--------|------| +| GET | /users | +`; + +describe('comment selector', () => { + it('matches block comments and not other html', () => { + expect(mdq(doc).query('comment').count()).toBe(2); + }); + + it('matches on the inner body so anchored patterns work', () => { + expect(mdq(doc).query('comment(/^test/)').count()).toBe(1); + }); + + it('exposes the inner body as node text, without the markers', () => { + expect(mdq(doc).query('comment[0]').nodes()[0].text).toBe('suite'); + }); + + it('keeps newlines inside a multi-line comment', () => { + expect(mdq(doc).query('comment(/^test/)').nodes()[0].text).toContain('\n'); + }); + + it('does not reach comments inline in a paragraph', () => { + expect(mdq(doc).query('comment(~"inline")').count()).toBe(0); + }); + + it('matches an exact single-line comment body', () => { + expect(mdq(doc).query('comment("suite")').count()).toBe(1); + }); +}); + +describe('html selector', () => { + it('matches every html block including comments', () => { + expect(mdq(doc).query('html').count()).toBe(3); + }); + + it('matches on raw text', () => { + expect(mdq(doc).query('html(~" { + it('honors an explicit i flag', () => { + expect(mdq('## Summary\n').query('h2(/^summary/i)').count()).toBe(1); + }); + + it('is case sensitive without the i flag', () => { + expect(mdq('## Summary\n').query('h2(/^summary/)').count()).toBe(0); + }); +}); + +describe('table text matching', () => { + it('matches cell content, not only headers', () => { + expect(mdq(doc).query('table(~"/users")').count()).toBe(1); + }); + + it('still matches header content', () => { + expect(mdq(doc).query('table(~"Method")').count()).toBe(1); + }); +}); + +describe('selector errors', () => { + it('throws on an unknown selector rather than matching nothing', () => { + expect(() => mdq(doc).query('secton("A")')).toThrow(MdqSelectorError); + }); + + it('reports where the problem is', () => { + try { + mdq(doc).query('h2("A") secton("B")'); + expect.unreachable(); + } catch (error) { + expect(error.index).toBe(7); + } + }); + + it('accepts a leading dot for jq muscle memory', () => { + expect(mdq('## A\n').query('.h2').count()).toBe(1); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test tests/unit/mdq/selectors.test.ts` +Expected: FAIL — `MdqSelectorError` is not exported + +- [ ] **Step 3: Implement the four changes** + +Error classes, at the top of the class section: + +```ts +export class MdqError extends Error {} + +export class MdqSelectorError extends MdqError { + index: number; + + constructor(message: string, index: number) { + super(message); + this.name = 'MdqSelectorError'; + this.index = index; + } +} +``` + +In `parseQuery`, skip one leading `.` per segment and throw on an unknown identifier. Replace the silent `pos++; continue;`: + +```ts + if (peek() === '.') advance(); + const selectorStart = pos; + const selector = readIdentifier(); + if (!selector) throw new MdqSelectorError(`Unexpected character "${input[pos]}" in selector`, pos); + if (!isKnownSelector(selector)) throw new MdqSelectorError(`Unknown selector "${selector}"`, selectorStart); +``` + +Add the vocabulary check and the two new token mappings: + +```ts +const SELECTORS = new Set(['section', 'heading', 'paragraph', 'table', 'list', 'item', 'code', 'blockquote', 'hr', 'html', 'comment']); + +function isKnownSelector(selector: string): boolean { + if (/^h[1-6]$/.test(selector)) return true; + if (/^section[1-6]?$/.test(selector)) return true; + return SELECTORS.has(selector); +} + +function isCommentToken(token: Token): boolean { + if (token.type !== 'html') return false; + return ((token as any).raw || '').trimStart().startsWith('$/, '').trim(); +} +``` + +In `selectorToTokenType`, map `html` to `'html'`. Handle `comment` in `executeSegments` before the generic branch, mirroring how `item` is handled: + +```ts + if (segment.selector === 'comment') { + let comments = candidates.filter((r) => isCommentToken(r.token)); + if (segment.textMatch) comments = comments.filter((r) => matchText(commentBody(r.token), segment.textMatch!)); + return executeSegments(applyIndexSlice(comments, segment), remaining); + } +``` + +In `getTokenText`, return the comment body for comment tokens, the raw for other html, and widen tables: + +```ts + case 'html': + if (isCommentToken(token)) return commentBody(token); + return t.raw || ''; + case 'table': + return [...(t.header || []).map((h: any) => h.text), ...(t.rows || []).flatMap((row: any) => row.map((cell: any) => cell.text))].join(', '); +``` + +In `parseTextMatcher`, capture the flags instead of discarding them: + +```ts + if (peek() === '/') { + advance(); + let value = ''; + while (pos < input.length && input[pos] !== '/') { + value += input[pos]; + pos++; + } + if (pos < input.length) pos++; + const flagStart = pos; + while (pos < input.length && /[gimsuy]/.test(input[pos])) pos++; + return { mode: 'regex', value, negated, flags: input.slice(flagStart, pos) }; + } +``` + +In `matchText`, use them: + +```ts + case 'regex': + result = new RegExp(matcher.value, matcher.flags || '').test(text); + break; +``` + +Add `flags?: string` to `TextMatcher` in the type block at the end of the file. + +Rename `meta()` to `nodes()` in the same pass, since the test above calls `nodes()`, and +keep `meta` as a deprecated alias. The return type gets a name now that it is public: + +```ts + nodes(): NodeInfo[] { + return this.matches.map((range) => { + const token = range.token as any; + if (token.type !== 'heading') return { type: token.type, depth: null, text: getTokenText(range.token) }; + return { type: token.type, depth: token.depth, text: getTokenText(range.token) }; + }); + } + + /** @deprecated Use nodes(). */ + meta(): NodeInfo[] { + return this.nodes(); + } +``` + +and at the end of the file: + +```ts +export interface NodeInfo { + type: string; + depth: number | null; + text: string; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test tests/unit/mdq/selectors.test.ts` +Expected: PASS (13 tests) + +- [ ] **Step 5: Verify nothing regressed** + +Run: `bun test tests/unit/` +Expected: PASS — in particular `query.test.ts` still at 114, since Task 3 already fixed `researcher.ts:316` + +- [ ] **Step 6: Format, lint and commit** + +```bash +bun run format && bun run lint:fix +git add -A src/utils/mdq tests/unit/mdq +git commit -m "feat(mdq): comment and html selectors, honest regex flags, loud selector errors" +``` + +--- + +### Task 5: Matchers as JS values, plus the sugar layer + +Removes the hand-escaping wart: `section.name.replace(/"/g, '\\"')` at `researcher/focus.ts:77` exists only because a matcher had to be embedded in a string. + +**Files:** +- Modify: `src/utils/mdq/query.ts` +- Create: `tests/unit/mdq/sugar.test.ts` + +**Interfaces:** +- Consumes: `MarkdownDoc`, `Selection` from Task 3; `MdqSelectorError` from Task 4 +- Produces: + - `type Matcher = string | RegExp | ((text: string) => boolean)` + - `interface SelectorOptions { depth?: 1 | 2 | 3 | 4 | 5 | 6 }` + - `query(selector: string, matcher?: Matcher): Selection` on both classes + - Sugar on both classes: `section` `heading` `paragraph` `table` `list` `item` `code` `blockquote` `comment` `html` `hr` + - `at(n: number): Selection` and `slice(from?: number, to?: number): Selection` on `Selection` + +- [ ] **Step 1: Write the failing test** + +```ts +// tests/unit/mdq/sugar.test.ts +import { describe, expect, it } from 'vitest'; +import { mdq } from '../../../src/utils/mdq/query.ts'; + +const doc = `## API "v2" + +| Method | Path | +|--------|------| +| GET | /users | + +## Settings + +- Option A +- Option B + + +`; + +describe('matchers as values', () => { + it('matches a string exactly', () => { + expect(mdq(doc).query('h2', 'Settings').count()).toBe(1); + expect(mdq(doc).query('h2', 'Setting').count()).toBe(0); + }); + + it('matches a RegExp honoring its flags', () => { + expect(mdq(doc).query('h2', /^settings$/i).count()).toBe(1); + expect(mdq(doc).query('h2', /^settings$/).count()).toBe(0); + }); + + it('matches a predicate', () => { + expect(mdq(doc).query('h2', (t) => t.startsWith('API')).count()).toBe(1); + }); + + it('needs no escaping for a value containing quotes', () => { + expect(mdq(doc).query('h2', 'API "v2"').count()).toBe(1); + }); +}); + +describe('sugar', () => { + it('is equivalent to the query form', () => { + expect(mdq(doc).heading('Settings').text()).toBe(mdq(doc).query('heading', 'Settings').text()); + }); + + it('takes a depth option', () => { + expect(mdq(doc).section('Settings', { depth: 2 }).text()).toBe(mdq(doc).query('section2("Settings")').text()); + }); + + it('reads comments', () => { + expect(mdq(doc).comment(/^test/).count()).toBe(1); + }); + + it('chains from a Selection', () => { + expect(mdq(doc).section('API "v2"').table().rows()[0].Path).toBe('/users'); + }); + + it('takes no matcher', () => { + expect(mdq(doc).table().count()).toBe(1); + }); +}); + +describe('at and slice', () => { + it('selects by index like the DSL', () => { + expect(mdq(doc).heading().at(0).text()).toBe(mdq(doc).query('heading[0]').text()); + }); + + it('supports a negative index', () => { + expect(mdq(doc).heading().at(-1).text()).toContain('Settings'); + }); + + it('returns nothing for an out-of-bounds index', () => { + expect(mdq(doc).heading().at(99).count()).toBe(0); + expect(mdq(doc).heading().at(-99).count()).toBe(0); + }); + + it('slices like the DSL', () => { + expect(mdq(doc).item().slice(1).count()).toBe(1); + }); +}); + +describe('exists', () => { + it('is true when something matched', () => { + expect(mdq(doc).heading('Settings').exists()).toBe(true); + }); + + it('is false when nothing matched', () => { + expect(mdq(doc).heading('Nope').exists()).toBe(false); + }); +}); + +describe('canonical read names', () => { + it('rows matches the deprecated toJson', () => { + expect(mdq(doc).table().rows()).toEqual(mdq(doc).table().toJson()); + }); + + it('preceding matches the deprecated before', () => { + expect(mdq(doc).heading('Settings').preceding().text()).toBe(mdq(doc).query('heading("Settings")').before().text()); + }); + + it('following matches the deprecated after', () => { + expect(mdq(doc).heading('API "v2"').following().text()).toBe(mdq(doc).query('heading(~"API")').after().text()); + }); + + it('entries matches the deprecated keyValue', () => { + const block = mdq('> Container: .x\n').query('blockquote[0]'); + expect(block.entries()).toEqual(block.keyValue()); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test tests/unit/mdq/sugar.test.ts` +Expected: FAIL — `mdq(...).heading is not a function` + +- [ ] **Step 3: Implement matchers and the shared sugar base** + +A value matcher bypasses the grammar entirely, so it needs its own `TextMatcher` mode: + +```ts +function toTextMatcher(matcher: Matcher): TextMatcher { + if (typeof matcher === 'function') return { mode: 'predicate', value: '', negated: false, predicate: matcher }; + if (matcher instanceof RegExp) return { mode: 'regex', value: matcher.source, negated: false, flags: matcher.flags }; + return { mode: 'exact', value: matcher, negated: false }; +} +``` + +In `matchText`, add the branch: + +```ts + case 'predicate': + result = matcher.predicate!(text); + break; +``` + +Both classes share the sugar through one abstract base. Write the eleven methods out explicitly rather than generating them on the prototype — generated methods lose their types, and a typed surface is the point of a public package: + +```ts +abstract class Queryable { + abstract query(selector: string, matcher?: Matcher): Selection; + + section(matcher?: Matcher, options?: SelectorOptions): Selection { + return this.query(`section${options?.depth || ''}`, matcher); + } + + heading(matcher?: Matcher, options?: SelectorOptions): Selection { + if (options?.depth) return this.query(`h${options.depth}`, matcher); + return this.query('heading', matcher); + } + + paragraph(matcher?: Matcher): Selection { + return this.query('paragraph', matcher); + } + + table(matcher?: Matcher): Selection { + return this.query('table', matcher); + } + + list(matcher?: Matcher): Selection { + return this.query('list', matcher); + } + + item(matcher?: Matcher): Selection { + return this.query('item', matcher); + } + + code(matcher?: Matcher): Selection { + return this.query('code', matcher); + } + + blockquote(matcher?: Matcher): Selection { + return this.query('blockquote', matcher); + } + + comment(matcher?: Matcher): Selection { + return this.query('comment', matcher); + } + + html(matcher?: Matcher): Selection { + return this.query('html', matcher); + } + + hr(): Selection { + return this.query('hr'); + } +} +``` + +`MarkdownDoc extends Queryable` and `Selection extends Queryable`. Each `query` applies the matcher to the last parsed segment: + +```ts + query(selector: string, matcher?: Matcher): Selection { + const segments = parseQuery(selector); + if (matcher !== undefined && segments.length > 0) segments[segments.length - 1].textMatch = toTextMatcher(matcher); + const candidates = expandSectionRanges(buildTokenIndex(this.source)); + return new Selection(this.source, executeSegments(candidates, segments)); + } +``` + +On `Selection`, add: + +```ts + at(index: number): Selection { + const resolved = index < 0 ? this.matches.length + index : index; + if (resolved < 0 || resolved >= this.matches.length) return new Selection(this.source, []); + return new Selection(this.source, [this.matches[resolved]]); + } + + slice(from?: number, to?: number): Selection { + return new Selection(this.source, this.matches.slice(from, to)); + } + + exists(): boolean { + return this.matches.length > 0; + } +``` + +At the end of the file add `Matcher`, `SelectorOptions`, `predicate?: (text: string) => boolean` on `TextMatcher`, and widen its `mode` to include `'predicate'`. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test tests/unit/mdq/sugar.test.ts` +Expected: PASS (21 tests) + +The `canonical read names` block fails until Step 5 adds the renames — that is expected. +Run Step 5 before treating those four as real failures. + +- [ ] **Step 5: Add the read renames and their deprecated aliases** + +Canonical names, with the old ones kept and marked: + +| Canonical | Deprecated alias | +|---|---| +| `text()` | `get()` | +| `rows()` | `toJson()` | +| `entries()` | `keyValue()` | +| `nodes()` | `meta()` (already added in Task 4) | +| `preceding()` | `before()` | +| `following()` | `after()` | + +Each alias is one line, for example: + +```ts + /** @deprecated Use rows(). */ + toJson(): Record[] { + return this.rows(); + } +``` + +Run: `bun test tests/unit/` +Expected: PASS — the ported suite still calls the deprecated names and must keep working + +- [ ] **Step 6: Format, lint and commit** + +```bash +bun run format && bun run lint:fix +git add -A src/utils/mdq tests/unit/mdq +git commit -m "feat(mdq): value matchers, sugar layer, at/slice, canonical read names" +``` + +--- + +### Task 6: edit.ts — remove and insert, with the whitespace invariant + +The first task in `edit.ts`, and the one most likely to produce subtly wrong output. The governing rule, from the spec: + +> **mdq never leaves zero blank lines between blocks, and never more than one.** + +This matters because `marked` separators are uneven: a `heading` raw is `"# A\n\n"` with its blank line baked in, while a `paragraph` raw is `"para"` with no newline at all and a sibling `space` token holding the `"\n\n"`. Task 1 recorded that sibling as `range.trailing`; every verb here uses it. + +**Files:** +- Create: `src/utils/mdq/edit.ts` +- Modify: `src/utils/mdq/query.ts` +- Create: `tests/unit/mdq/edit.test.ts` + +**Interfaces:** +- Consumes: `MatchedRange` (type-only) from Task 1; `MarkdownDoc`, `Selection` from Task 3 +- Produces, all in `edit.ts`: + - `spliceRanges(source: string, ranges: MatchedRange[], render: (range: MatchedRange, index: number) => string): string` + - `removeRanges(source: string, ranges: MatchedRange[]): string` + - `insertAt(source: string, offset: number, markdown: string): string` + - `blockEnd(range: MatchedRange): number` and `blockStart(range: MatchedRange): number` +- Produces, on `Selection`: `remove()`, `insertBefore(md)`, `insertAfter(md)`, `prepend(md)`, `append(md)` +- Produces, on `MarkdownDoc`: `append(md)`, `prepend(md)` + +- [ ] **Step 1: Write the failing test** + +```ts +// tests/unit/mdq/edit.test.ts +import { describe, expect, it } from 'vitest'; +import { mdq } from '../../../src/utils/mdq/query.ts'; + +describe('remove', () => { + it('takes a paragraph and its separator, leaving no crater', () => { + expect(mdq('# A\n\nfirst\n\nsecond\n').query('paragraph("first")').remove().toString()).toBe('# A\n\nsecond\n'); + }); + + it('takes a heading with its baked-in separator', () => { + expect(mdq('# A\n\n## B\n\ntext\n').query('h2').remove().toString()).toBe('# A\n\ntext\n'); + }); + + it('takes the leading separator when the node is last', () => { + expect(mdq('# A\n\nlast\n').query('paragraph').remove().toString()).toBe('# A\n'); + }); + + it('removes a whole section including its children', () => { + expect(mdq('## A\n\nx\n\n## B\n\ny\n').query('section("A")').remove().toString()).toBe('## B\n\ny\n'); + }); + + it('removes every match', () => { + expect(mdq('# T\n\n```js\na\n```\n\ntext\n\n```js\nb\n```\n').query('code').remove().toString()).toBe('# T\n\ntext\n'); + }); + + it('returns the document unchanged when nothing matches', () => { + const src = '# A\n\ntext\n'; + expect(mdq(src).query('h5').remove().toString()).toBe(src); + }); +}); + +describe('insertBefore and insertAfter', () => { + it('inserts a sibling before a node', () => { + expect(mdq('## B\n\ntext\n').query('h2').insertBefore('## A\n').toString()).toBe('## A\n\n## B\n\ntext\n'); + }); + + it('inserts a sibling after a node', () => { + expect(mdq('## A\n\ntext\n').query('h2').insertAfter('## B\n').toString()).toBe('## A\n\n## B\n\ntext\n'); + }); + + it('normalizes an insert that already ends with blank lines', () => { + expect(mdq('## A\n\ntext\n').query('h2').insertAfter('## B\n\n\n\n').toString()).toBe('## A\n\n## B\n\ntext\n'); + }); + + it('normalizes an insert with no trailing newline', () => { + expect(mdq('## A\n\ntext\n').query('h2').insertAfter('## B').toString()).toBe('## A\n\n## B\n\ntext\n'); + }); + + it('accepts a MarkdownDoc', () => { + const fragment = mdq('## B\n'); + expect(mdq('## A\n\ntext\n').query('h2').insertAfter(fragment).toString()).toContain('## B'); + }); +}); + +describe('prepend and append on a section', () => { + const src = '## A\n\nfirst\n\n## B\n\nother\n'; + + it('appends inside the section, before the next same-depth heading', () => { + expect(mdq(src).query('section("A")').append('last\n').toString()).toBe('## A\n\nfirst\n\nlast\n\n## B\n\nother\n'); + }); + + it('prepends directly after the section heading', () => { + expect(mdq(src).query('section("A")').prepend('intro\n').toString()).toBe('## A\n\nintro\n\nfirst\n\n## B\n\nother\n'); + }); + + it('appends at the end of the document when the section is last', () => { + expect(mdq(src).query('section("B")').append('tail\n').toString()).toBe('## A\n\nfirst\n\n## B\n\nother\n\ntail\n'); + }); + + it('throws when applied to a leaf node', () => { + expect(() => mdq(src).query('paragraph[0]').append('x\n')).toThrow(); + }); +}); + +describe('document-level append and prepend', () => { + it('appends a block at the end', () => { + expect(mdq('# A\n\ntext\n').append('## New\n').toString()).toBe('# A\n\ntext\n\n## New\n'); + }); + + it('prepends a block at the start', () => { + expect(mdq('# A\n\ntext\n').prepend('> note\n').toString()).toBe('> note\n\n# A\n\ntext\n'); + }); + + it('prepends after frontmatter, never before it', () => { + const out = mdq('---\nurl: /x\n---\n\n# A\n').prepend('> note\n').toString(); + expect(out.startsWith('---\nurl: /x\n---\n')).toBe(true); + expect(out).toContain('> note'); + }); +}); + +describe('chained edits', () => { + it('composes several writes in one expression', () => { + const out = mdq('## A\n\nfirst\n\n## B\n\nother\n') + .query('section("A")') + .append('added\n') + .query('paragraph("other")') + .remove() + .toString(); + expect(out).toBe('## A\n\nfirst\n\nadded\n\n## B\n'); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test tests/unit/mdq/edit.test.ts` +Expected: FAIL — `.remove is not a function` + +- [ ] **Step 3: Write edit.ts** + +```ts +import type { MatchedRange } from './query.ts'; + +export function blockStart(range: MatchedRange): number { + return range.start; +} + +export function blockEnd(range: MatchedRange): number { + if (range.trailing) return range.trailing.start + range.trailing.length; + return range.start + range.length; +} + +export function normalizeBlock(markdown: string): string { + return `${markdown.replace(/\s+$/, '')}\n`; +} + +export function spliceRanges(source: string, ranges: MatchedRange[], render: (range: MatchedRange, index: number) => string): string { + const ordered = dedupeRanges(ranges); + const rendered = ordered.map(render); + let result = source; + for (let i = ordered.length - 1; i >= 0; i--) { + const range = ordered[i]; + result = result.slice(0, range.start) + rendered[i] + result.slice(range.start + range.length); + } + return result; +} + +export function removeRanges(source: string, ranges: MatchedRange[]): string { + const ordered = dedupeRanges(ranges); + let result = source; + for (let i = ordered.length - 1; i >= 0; i--) { + const range = ordered[i]; + const end = blockEnd(range); + let start = range.start; + if (!range.trailing) start = trimPrecedingBlankLine(result, start); + result = result.slice(0, start) + result.slice(end); + } + return result; +} + +export function insertAt(source: string, offset: number, markdown: string): string { + const block = normalizeBlock(markdown); + const before = source.slice(0, offset); + const after = source.slice(offset); + if (!after) return `${before}${before.endsWith('\n') ? '' : '\n'}\n${block}`.replace(/\n{3,}/g, '\n\n'); + return `${before}${block}\n${after}`.replace(/\n{3,}/g, '\n\n'); +} + +function dedupeRanges(ranges: MatchedRange[]): MatchedRange[] { + const sorted = [...ranges].sort((a, b) => a.start - b.start); + const kept: MatchedRange[] = []; + let lastEnd = -1; + for (const range of sorted) { + if (range.start < lastEnd) continue; + kept.push(range); + lastEnd = range.start + range.length; + } + return kept; +} + +function trimPrecedingBlankLine(source: string, start: number): number { + let cursor = start; + while (cursor > 0 && source[cursor - 1] === '\n') cursor--; + if (cursor === 0) return 0; + return cursor + 1; +} +``` + +Note `insertAt` collapses any run of three or more newlines to exactly two. That single rule is what enforces the invariant across every insert path, rather than each verb reasoning about separators itself. + +- [ ] **Step 4: Wire the verbs onto Selection and MarkdownDoc** + +On `Selection`, four public verbs delegating to one private helper: + +```ts + remove(): MarkdownDoc { + return new MarkdownDoc(removeRanges(this.source, this.matches)); + } + + insertBefore(markdown: Markdown): MarkdownDoc { + return this.insertEach((range) => blockStart(range), markdown); + } + + insertAfter(markdown: Markdown): MarkdownDoc { + return this.insertEach((range) => blockEnd(range), markdown); + } + + prepend(markdown: Markdown): MarkdownDoc { + return this.insertEach((range) => this.containerStart(range), markdown); + } + + append(markdown: Markdown): MarkdownDoc { + return this.insertEach((range) => this.containerEnd(range), markdown); + } +``` + +`insertEach`, `containerStart` and `containerEnd` are private and placed after the public +methods. Inserts run back-to-front so earlier offsets stay valid: + +```ts + private insertEach(offsetOf: (range: MatchedRange) => number, markdown: Markdown): MarkdownDoc { + const offsets = this.matches.map(offsetOf).sort((a, b) => a - b); + let result = this.source; + for (let i = offsets.length - 1; i >= 0; i--) { + result = insertAt(result, offsets[i], String(markdown)); + } + return new MarkdownDoc(result); + } + + private containerStart(range: MatchedRange): number { + if (!range.innerTokens) throw new MdqOperationError(`prepend needs a section or list, got ${range.token.type}`); + return range.start + ((range.token as any).raw || '').length; + } + + private containerEnd(range: MatchedRange): number { + if (!range.innerTokens) throw new MdqOperationError(`append needs a section or list, got ${range.token.type}`); + const last = range.innerTokens[range.innerTokens.length - 1]; + if (!last) return this.containerStart(range); + return blockEnd(last); + } +``` + +`containerStart` and `containerEnd` throw eagerly, which is why the "throws when applied to +a leaf node" test asserts on the `query(...).append(...)` call itself rather than on +`.toString()`. + +On `MarkdownDoc`: + +```ts + append(markdown: Markdown): MarkdownDoc { + return new MarkdownDoc(insertAt(this.source, this.source.length, String(markdown))); + } + + prepend(markdown: Markdown): MarkdownDoc { + return new MarkdownDoc(insertAt(this.source, splitFrontmatter(this.source).offset, String(markdown))); + } +``` + +Add `MdqOperationError`: + +```ts +export class MdqOperationError extends MdqError { + constructor(message: string) { + super(message); + this.name = 'MdqOperationError'; + } +} +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `bun test tests/unit/mdq/edit.test.ts` +Expected: PASS (19 tests) + +If a whitespace assertion fails, print the actual output with `JSON.stringify` before changing anything — the difference is almost always one newline, and guessing at it will break a different case. + +- [ ] **Step 6: Run the whole suite and commit** + +```bash +bun test tests/unit/ +bun run format && bun run lint:fix +git add -A src/utils/mdq tests/unit/mdq +git commit -m "feat(mdq): remove and insert verbs with whitespace normalization" +``` + +--- + +### Task 7: Structural inserts — addRow and addItem + +**Files:** +- Modify: `src/utils/mdq/edit.ts` +- Modify: `src/utils/mdq/query.ts` +- Create: `tests/unit/mdq/structural.test.ts` + +**Interfaces:** +- Consumes: `spliceRanges` from Task 6; `MdqOperationError` from Task 6 +- Produces in `edit.ts`: `renderTable(headers: string[], rows: string[][], align: (string | null)[]): string`, `renderItem(listRaw: string, text: string): string` +- Produces on `Selection`: `addRow(row: Record): MarkdownDoc`, `addItem(text: string): MarkdownDoc` + +- [ ] **Step 1: Write the failing test** + +```ts +// tests/unit/mdq/structural.test.ts +import { describe, expect, it } from 'vitest'; +import { mdq } from '../../../src/utils/mdq/query.ts'; + +const table = `| Method | Path | +|--------|------| +| GET | /users | +`; + +describe('addRow', () => { + it('appends a row and re-aligns every column', () => { + expect(mdq(table).query('table').addRow({ Method: 'POST', Path: '/sessions' }).toString()).toBe( + ['| Method | Path |', '| ------ | --------- |', '| GET | /users |', '| POST | /sessions |', ''].join('\n') + ); + }); + + it('round-trips through rows()', () => { + const out = mdq(table).query('table').addRow({ Method: 'POST', Path: '/sessions' }); + expect(mdq(out).query('table').rows()).toEqual([ + { Method: 'GET', Path: '/users' }, + { Method: 'POST', Path: '/sessions' }, + ]); + }); + + it('leaves a column blank when the object omits it', () => { + const out = mdq(table).query('table').addRow({ Method: 'PUT' }); + expect(mdq(out).query('table').rows()[1]).toEqual({ Method: 'PUT', Path: '' }); + }); + + it('ignores keys that are not columns', () => { + const out = mdq(table).query('table').addRow({ Method: 'PUT', Nope: 'x' }); + expect(mdq(out).query('table').rows()[1].Method).toBe('PUT'); + expect(mdq(out).query('table').text()).not.toContain('Nope'); + }); + + it('throws on a non-table node', () => { + expect(() => mdq('para\n').query('paragraph').addRow({ a: 'b' })).toThrow(); + }); +}); + +describe('addItem', () => { + it('copies a dash marker', () => { + expect(mdq('- a\n- b\n').query('list').addItem('c').toString()).toBe('- a\n- b\n- c\n'); + }); + + it('copies a star marker', () => { + expect(mdq('* a\n* b\n').query('list').addItem('c').toString()).toBe('* a\n* b\n* c\n'); + }); + + it('continues an ordered list', () => { + expect(mdq('1. a\n2. b\n').query('list').addItem('c').toString()).toBe('1. a\n2. b\n3. c\n'); + }); + + it('preserves indentation', () => { + expect(mdq(' - a\n - b\n').query('list').addItem('c').toString()).toBe(' - a\n - b\n - c\n'); + }); + + it('throws on a non-list node', () => { + expect(() => mdq('para\n').query('paragraph').addItem('x')).toThrow(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test tests/unit/mdq/structural.test.ts` +Expected: FAIL — `.addRow is not a function` + +- [ ] **Step 3: Add the renderers to edit.ts** + +```ts +export function renderTable(headers: string[], rows: string[][], align: (string | null)[]): string { + const widths = headers.map((header, index) => Math.max(header.length, 3, ...rows.map((row) => (row[index] || '').length))); + const line = (cells: string[]) => `| ${cells.map((cell, index) => (cell || '').padEnd(widths[index])).join(' | ')} |`; + const divider = `| ${widths.map((width, index) => dashes(align[index], width)).join(' | ')} |`; + return `${[line(headers), divider, ...rows.map(line)].join('\n')}\n`; +} + +export function renderItem(listRaw: string, text: string): string { + const lines = listRaw.split('\n').filter((line) => line.trim()); + const last = lines[lines.length - 1] || '- x'; + const match = last.match(/^(\s*)(\d+)([.)])\s/); + if (match) return `${match[1]}${Number.parseInt(match[2], 10) + 1}${match[3]} ${text}`; + const bullet = last.match(/^(\s*)([-*+])\s/); + if (!bullet) return `- ${text}`; + return `${bullet[1]}${bullet[2]} ${text}`; +} + +function dashes(alignment: string | null, width: number): string { + if (alignment === 'center') return `:${'-'.repeat(Math.max(width - 2, 1))}:`; + if (alignment === 'left') return `:${'-'.repeat(Math.max(width - 1, 1))}`; + if (alignment === 'right') return `${'-'.repeat(Math.max(width - 1, 1))}:`; + return '-'.repeat(width); +} +``` + +- [ ] **Step 4: Wire the verbs onto Selection** + +```ts + addRow(row: Record): MarkdownDoc { + return new MarkdownDoc( + spliceRanges(this.source, this.matches, (range) => { + if (range.token.type !== 'table') throw new MdqOperationError(`addRow needs a table, got ${range.token.type}`); + const table = range.token as Tokens.Table; + const headers = table.header.map((cell) => cell.text); + const existing = table.rows.map((cells) => headers.map((_, index) => cells[index]?.text || '')); + return renderTable(headers, [...existing, headers.map((header) => row[header] || '')], table.align); + }) + ); + } + + addItem(text: string): MarkdownDoc { + return new MarkdownDoc( + spliceRanges(this.source, this.matches, (range) => { + if (range.token.type !== 'list') throw new MdqOperationError(`addItem needs a list, got ${range.token.type}`); + const raw = ((range.token as any).raw || '').replace(/\s+$/, ''); + return `${raw}\n${renderItem(raw, text)}\n`; + }) + ); + } +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `bun test tests/unit/mdq/structural.test.ts` +Expected: PASS (10 tests) + +- [ ] **Step 6: Run the whole suite and commit** + +```bash +bun test tests/unit/ +bun run format && bun run lint:fix +git add -A src/utils/mdq tests/unit/mdq +git commit -m "feat(mdq): addRow and addItem structural inserts" +``` + +--- + +### Task 8: setEntry and the frontmatter API + +**Files:** +- Modify: `src/utils/mdq/edit.ts` +- Modify: `src/utils/mdq/query.ts` +- Modify: `tests/unit/mdq/frontmatter.test.ts` + +**Interfaces:** +- Consumes: `splitFrontmatter` from Task 1; `spliceRanges` from Task 6 +- Produces in `edit.ts`: `rewriteEntries(tokenText: string, isBlockquote: boolean, key: string, value: string | null): string`, `writeFrontmatter(source: string, key: string, value: unknown): string` +- Produces on `Selection`: `entries()`, `setEntry(key, value)` +- Produces on `MarkdownDoc`: `frontmatter(): Record`, `setFrontmatter(key: string, value: unknown): MarkdownDoc` + +Reading and writing both go through `yaml`'s **Document API** (`YAML.parseDocument`), never `parse`/`stringify`. That is what preserves comments through a write — verified behaviour, not an assumption. + +- [ ] **Step 1: Write the failing test** + +```ts +// append to tests/unit/mdq/frontmatter.test.ts +import { mdq } from '../../../src/utils/mdq/query.ts'; + +describe('frontmatter API', () => { + const src = '---\n# a leading comment\nurl: /login\nwait: 1000\ntags:\n - auth\n - smoke\n---\n\n# Title\n'; + + it('reads typed scalars, lists and nested maps', () => { + expect(mdq(src).frontmatter()).toEqual({ url: '/login', wait: 1000, tags: ['auth', 'smoke'] }); + }); + + it('returns an empty object when there is no frontmatter', () => { + expect(mdq('# Title\n').frontmatter()).toEqual({}); + }); + + it('updates a key in place', () => { + expect(mdq(src).setFrontmatter('wait', 2000).frontmatter().wait).toBe(2000); + }); + + it('preserves comments through a write', () => { + expect(mdq(src).setFrontmatter('wait', 2000).toString()).toContain('# a leading comment'); + }); + + it('preserves the body exactly', () => { + expect(mdq(src).setFrontmatter('wait', 2000).toString()).toContain('# Title'); + }); + + it('adds a key that was not there', () => { + expect(mdq(src).setFrontmatter('region', 'sidebar').frontmatter().region).toBe('sidebar'); + }); + + it('deletes a key when the value is null', () => { + expect(mdq(src).setFrontmatter('wait', null).frontmatter().wait).toBeUndefined(); + }); + + it('creates a frontmatter block on a document that has none', () => { + const out = mdq('# Title\n').setFrontmatter('url', '/x'); + expect(out.frontmatter()).toEqual({ url: '/x' }); + expect(out.toString()).toContain('# Title'); + }); + + it('keeps body queries blind to frontmatter after a write', () => { + expect(mdq(src).setFrontmatter('wait', 2000).query('h2').count()).toBe(0); + }); +}); + +describe('entries and setEntry', () => { + const block = "## S\n\n> Container: '.old'\n> Pagination: controls\n\ntext\n"; + + it('reads every entry of a blockquote without its markers', () => { + expect(mdq(block).query('blockquote[0]').entries()).toEqual({ container: "'.old'", pagination: 'controls' }); + }); + + it('replaces an entry in place and keeps the others', () => { + expect(mdq(block).query('blockquote[0]').setEntry('Container', "'.new'").toString()).toBe("## S\n\n> Container: '.new'\n> Pagination: controls\n\ntext\n"); + }); + + it('appends an entry that was not there', () => { + const out = mdq(block).query('blockquote[0]').setEntry('Region', 'sidebar'); + expect(mdq(out).query('blockquote[0]').entries().region).toBe('sidebar'); + }); + + it('removes an entry when the value is null', () => { + const out = mdq(block).query('blockquote[0]').setEntry('Pagination', null); + expect(mdq(out).query('blockquote[0]').entries()).toEqual({ container: "'.old'" }); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test tests/unit/mdq/frontmatter.test.ts` +Expected: FAIL — `.frontmatter is not a function` + +- [ ] **Step 3: Implement in edit.ts** + +```ts +import YAML from 'yaml'; + +export function writeFrontmatter(source: string, key: string, value: unknown): string { + const { raw, body, offset } = splitFrontmatter(source); + const document = YAML.parseDocument(raw || ''); + if (value === null) document.delete(key); + if (value !== null) document.set(key, value); + const rendered = document.toString().replace(/\s+$/, ''); + if (!offset) return `---\n${rendered}\n---\n\n${source}`; + return `---\n${rendered}\n---\n${body}`; +} + +export function readFrontmatter(source: string): Record { + const { raw } = splitFrontmatter(source); + if (!raw) return {}; + return (YAML.parseDocument(raw).toJS() as Record) || {}; +} +``` + +`splitFrontmatter` already lives in `edit.ts` from Task 1, so these functions sit beside it. + +`rewriteEntries` is the existing `setKeyValue` body, lifted out of the class: + +```ts +export function rewriteEntries(tokenText: string, isBlockquote: boolean, key: string, value: string | null): string { + const lines = tokenText + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); + + const index = lines.findIndex((line) => entryKey(line) === key.toLowerCase()); + if (index < 0 && value) lines.push(`${key}: ${value}`); + if (index >= 0 && value) lines[index] = `${key}: ${value}`; + if (index >= 0 && !value) lines.splice(index, 1); + + if (!isBlockquote) return lines.join('\n'); + return lines.map((line) => `> ${line}`).join('\n'); +} +``` + +`entryKey` moves to `edit.ts` alongside it. + +- [ ] **Step 4: Wire onto the classes** + +```ts + frontmatter(): Record { + return readFrontmatter(this.source); + } + + setFrontmatter(key: string, value: unknown): MarkdownDoc { + return new MarkdownDoc(writeFrontmatter(this.source, key, value)); + } +``` + +```ts + setEntry(key: string, value: string | null): MarkdownDoc { + return new MarkdownDoc(spliceRanges(this.source, this.matches, (range) => rewriteEntries(getTokenText(range.token), range.token.type === 'blockquote', key, value))); + } + + /** @deprecated Use setEntry(). */ + setKeyValue(key: string, value: string | null): MarkdownDoc { + return this.setEntry(key, value); + } +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `bun test tests/unit/mdq/frontmatter.test.ts` +Expected: PASS (21 tests) + +- [ ] **Step 6: Confirm knowledge and experience files now parse correctly** + +This is the real-world check that motivated the feature: + +```bash +bun -e ' +import { mdq } from "./src/utils/mdq/query.ts"; +import { readdirSync, readFileSync } from "node:fs"; +for (const dir of ["knowledge", "experience"]) { + for (const file of readdirSync(dir).filter((f) => f.endsWith(".md")).slice(0, 5)) { + const doc = mdq(readFileSync(`${dir}/${file}`, "utf8")); + console.log(file, JSON.stringify(doc.frontmatter()), "headings:", doc.query("heading").count()); + } +}' +``` + +Expected: frontmatter parsed as an object on each file, and **no heading whose text looks like `url: ...`**. A `url:` heading means frontmatter is leaking into the token index. + +If either directory is empty, skip this step and note it. + +- [ ] **Step 7: Run the whole suite and commit** + +```bash +bun test tests/unit/ +bun run format && bun run lint:fix +git add -A src/utils/mdq tests/unit/mdq +git commit -m "feat(mdq): frontmatter read/write via yaml Document API, setEntry" +``` + +--- + +### Task 9: README + +The package is publish-ready only if someone can use it without reading the source. + +**Files:** +- Create: `src/utils/mdq/README.md` + +**Interfaces:** +- Consumes: the complete API from Tasks 3-8 +- Produces: nothing code depends on + +- [ ] **Step 1: Write the README** + +Cover, in this order: + +1. One-paragraph pitch: query and edit markdown with a selector language, like jq for markdown. +2. Install and import. +3. **The one rule**, stated early and plainly: *reads narrow, writes return the document.* +4. Selector grammar table: `section` `section1-6` `h1-h6` `heading` `paragraph` `table` `list` `item` `code` `blockquote` `hr` `html` `comment`, with text matchers (`"exact"`, `~"contains"`, `/regex/flags`, `!` to negate), `[index]`, `[from:to]`, and compound paths. +5. Matchers as values: `string` exact, `RegExp` with its own flags, predicate function. +6. Read methods table, write methods table. +7. Frontmatter section, noting comment preservation. +8. A worked example using a chained multi-edit. +9. Limitations, stated honestly: block-level comments only (inline comments live inside paragraph tokens); no row- or item-level selectors, so `addRow` has no `removeRow` partner; YAML frontmatter only. + +Do **not** document the deprecated aliases (`get` `toJson` `keyValue` `setKeyValue` `meta` `before` `after`). They exist for in-repo callers; the published surface should read clean. + +Follow the repo docs style: show each format example once, and do not close with a "Why this matters" section. + +- [ ] **Step 2: Verify every example in the README actually runs** + +Extract each fenced `js` block and execute it. Any example that throws or prints something other than what the README claims is a documentation bug — fix the README, not the test. + +```bash +bun -e ' +import { mdq } from "./src/utils/mdq/query.ts"; +// paste each README example here and assert its stated output +' +``` + +- [ ] **Step 3: Commit** + +```bash +git add src/utils/mdq/README.md +git commit -m "docs(mdq): package README" +``` + +--- + +### Task 10: The CLI + +**Files:** +- Create: `src/utils/mdq/cli.ts` +- Create: `bin/mdq.ts` +- Create: `tests/unit/mdq/cli.test.ts` +- Modify: `package.json` (add the `mdq` bin entry) + +**Interfaces:** +- Consumes: the full library API from Tasks 3-8 +- Produces: `runMdq(argv: string[], stdin: string): Promise<{ output: string; code: number }>` + +`runMdq` returns its result rather than writing to stdout or calling `process.exit`, which is what makes it testable. `bin/mdq.ts` is the only place that touches the process. + +Note a deliberate deviation from `CLAUDE.md`: command logic normally lives in `src/commands/`, but mdq must not import from anywhere in explorbot. Its CLI ships with the package. + +- [ ] **Step 1: Write the failing test** + +```ts +// tests/unit/mdq/cli.test.ts +import { describe, expect, it } from 'vitest'; +import { runMdq } from '../../../src/utils/mdq/cli.ts'; + +const doc = `# Title + +## API + +| Method | Path | +|--------|------| +| GET | /users | + +## FAQ + +question? +`; + +describe('reads', () => { + it('prints matched markdown', async () => { + const result = await runMdq(['h2'], doc); + expect(result.output).toContain('## API'); + expect(result.code).toBe(0); + }); + + it('accepts a leading dot like jq', async () => { + expect((await runMdq(['.h2'], doc)).output).toContain('## API'); + }); + + it('prints rows as json', async () => { + const result = await runMdq(['section("API") table', '--json'], doc); + expect(JSON.parse(result.output)).toEqual([{ Method: 'GET', Path: '/users' }]); + }); + + it('prints a count', async () => { + expect((await runMdq(['h2', '--count'], doc)).output.trim()).toBe('2'); + }); + + it('prints unwrapped text', async () => { + expect((await runMdq(['h2', '--text'], doc)).output).not.toContain('##'); + }); + + it('prints frontmatter as json', async () => { + const result = await runMdq(['--frontmatter'], '---\nurl: /x\n---\n\n# T\n'); + expect(JSON.parse(result.output)).toEqual({ url: '/x' }); + }); +}); + +describe('edits', () => { + it('removes and prints the whole document', async () => { + const result = await runMdq(['section("FAQ")', '--remove'], doc); + expect(result.output).not.toContain('## FAQ'); + expect(result.output).toContain('## API'); + }); + + it('appends into a section', async () => { + expect((await runMdq(['section("FAQ")', '--append', 'answer!'], doc)).output).toContain('answer!'); + }); + + it('adds a table row from json', async () => { + const result = await runMdq(['table', '--add-row', '{"Method":"POST","Path":"/s"}'], doc); + expect(result.output).toContain('POST'); + }); + + it('sets an entry', async () => { + expect((await runMdq(['blockquote', '--set', 'Container=.x'], '> Container: .old\n')).output).toContain('.x'); + }); +}); + +describe('exit codes', () => { + it('returns 1 when nothing matches', async () => { + expect((await runMdq(['h5'], doc)).code).toBe(1); + }); + + it('returns 2 on an unknown selector', async () => { + const result = await runMdq(['secton("A")'], doc); + expect(result.code).toBe(2); + expect(result.output).toContain('Unknown selector'); + }); + + it('returns 0 when an edit matched', async () => { + expect((await runMdq(['h2', '--remove'], doc)).code).toBe(0); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test tests/unit/mdq/cli.test.ts` +Expected: FAIL — cannot resolve `cli.ts` + +- [ ] **Step 3: Implement cli.ts** + +Use Commander with `exitOverride()` and `.configureOutput()` so a parse failure surfaces as a return value rather than killing the process. Shape: + +```ts +export async function runMdq(argv: string[], stdin: string): Promise { + const program = new Command(); + program + .argument('[selector]', 'markdown selector') + .argument('[file]', 'file to read; stdin when omitted') + .option('-j, --json', 'output rows as JSON') + .option('-c, --count', 'print the number of matches') + .option('-t, --text', 'print unwrapped text') + .option('--frontmatter', 'print frontmatter as JSON') + .option('-i, --in-place', 'write the result back to the file') + .option('--remove', 'delete matched blocks') + .option('--replace ', 'replace matched blocks') + .option('--insert-before ', 'insert before each match') + .option('--insert-after ', 'insert after each match') + .option('--prepend ', 'insert at the start of each match') + .option('--append ', 'insert at the end of each match') + .option('--add-row ', 'append a table row') + .option('--add-item ', 'append a list item') + .option('--set ', 'set an entry; omit the value to delete it') + .exitOverride(); + // ... parse, read source, build doc, apply exactly one edit or one read, return { output, code } +} +``` + +Rules to implement: + +- No selector plus `--frontmatter` prints the frontmatter and returns 0. +- An edit flag with no matches returns 1 and prints the document unchanged. +- `MdqSelectorError` returns 2 with the message as output. +- `--in-place` writes `output` to the file and returns an empty `output`. +- More than one edit flag returns 2 with `Only one edit at a time`. +- `--set k=v` splits on the **first** `=`; `--set k=` deletes. + +At the end of the file: + +```ts +export interface CliResult { + output: string; + code: number; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test tests/unit/mdq/cli.test.ts` +Expected: PASS (13 tests) + +- [ ] **Step 5: Add the thin bin entry** + +```ts +#!/usr/bin/env bun +import { readFileSync } from 'node:fs'; +import { runMdq } from '../src/utils/mdq/cli.ts'; + +const stdin = process.stdin.isTTY ? '' : readFileSync(0, 'utf8'); +const result = await runMdq(process.argv.slice(2), stdin); +if (result.output) process.stdout.write(result.output.endsWith('\n') ? result.output : `${result.output}\n`); +process.exit(result.code); +``` + +Add to `package.json` `bin`: + +```json + "mdq": "./dist/bin/mdq.js" +``` + +- [ ] **Step 6: Smoke-test the real binary** + +```bash +echo '# A + +## B + +text' | bun run bin/mdq.ts 'h2' +``` +Expected: `## B` + +```bash +bun run bin/mdq.ts 'section("Data Envelope Formats") table' --json CLAUDE.md | head -5 +``` +Expected: JSON array of that section's table rows + +```bash +bun run bin/mdq.ts 'nonsense' CLAUDE.md; echo "exit=$?" +``` +Expected: `Unknown selector "nonsense"` and `exit=2` + +- [ ] **Step 7: Run the whole suite and commit** + +```bash +bun test tests/unit/ +bun run format && bun run lint:fix +git add -A src/utils/mdq bin/mdq.ts tests/unit/mdq package.json +git commit -m "feat(mdq): jq-like CLI" +``` + +--- + +### Task 11: Changelog and final verification + +**Files:** +- Modify: `CHANGELOG.md` + +- [ ] **Step 1: Run the full unit suite** + +Run: `bun test tests/unit/` +Expected: PASS, no failures + +- [ ] **Step 2: Re-run the scoped type check** + +```bash +bunx tsc -p tsconfig.json --noEmit 2>&1 | grep -E "^(src/utils/mdq/|src/utils/markdown-query|src/experience-tracker|src/ai/planner|src/ai/researcher|bin/mdq)" +``` + +Expected: **exactly the two known `locators.ts(247,...)` lines.** Anything else is a real defect that CI will not catch. + +- [ ] **Step 3: Confirm the package has no explorbot imports** + +```bash +grep -rn "^import\|from '" src/utils/mdq/*.ts | grep -v "'marked'" | grep -v "'yaml'" | grep -v "'commander'" | grep -v "'./" +``` + +Expected: **no output.** Any line here breaks extractability, which is the whole point of the package. + +- [ ] **Step 4: Update the changelog** + +Use the `/changelog` skill, per `CLAUDE.md`. + +- [ ] **Step 5: Commit** + +```bash +git add CHANGELOG.md +git commit -m "docs: changelog for mdq package" +``` + +--- + +## Notes for the executor + +- **Never run the regression workflow.** Do not add the `regression` label, do not `gh workflow run regression.yml`, do not re-run its jobs. If a change needs regression coverage, say so and let the user decide. +- **`.claude/worktrees/**` holds four stale copies** of `markdown-query.ts` and its consumers. Exclude that path from every grep, sed and sweep. A migration that "finds" 200 call sites has picked up worktrees. +- **CI type-checking is a mirage.** `tsc` runs with `--noCheck`, so a green build says nothing about the return-type change in Task 3. The scoped check in Task 3 Step 8 and Task 11 Step 2 is the only real gate. +- **Whitespace failures are one newline.** When an edit test fails, print the actual string with `JSON.stringify` before touching the implementation. Adjusting the normalizer by guesswork fixes one case and breaks two. From 791348c4ff66953866b8c078d4e8eae2c5c8ad63 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:12:49 +0300 Subject: [PATCH 04/21] docs: correct whitespace rules in mdq plan from token spikes removeRanges collapsed backwards whenever a node had no trailing space token, which ate a blank line on '# A\n\n## B\n\ntext\n'. marked bakes separators into heading and document-final paragraph raws, so the rule is "collapse only at end-of-document". insertAt used a global \n{3,} collapse that would rewrite blank lines inside fenced code blocks; scoped it to the seam. Also: selector error index is 8 not 7, YAML needs new Document({}) when frontmatter is absent, and the --set CLI test now proves the old value is gone. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- .../plans/2026-09-15-mdq-package.md | 66 ++++++++++++++----- 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/docs/superpowers/plans/2026-09-15-mdq-package.md b/docs/superpowers/plans/2026-09-15-mdq-package.md index ecedbe44..a826755b 100644 --- a/docs/superpowers/plans/2026-09-15-mdq-package.md +++ b/docs/superpowers/plans/2026-09-15-mdq-package.md @@ -411,6 +411,9 @@ export function mdq(source: Markdown): MarkdownDoc { } ``` +`query` takes only a selector here. The optional second `matcher` argument arrives in +Task 5 — it is not missing. + At the end of the file: ```ts @@ -420,6 +423,14 @@ export type Markdown = string | MarkdownDoc; export const MarkdownQuery = Selection; ``` +Before relying on that alias, confirm nothing imports the class by name: + +```bash +git grep -ln "MarkdownQuery" -- src bin boat tests +``` + +Expected: only `src/utils/mdq/query.ts`. Any other file needs its import checked. + - [ ] **Step 4: Run test to verify it passes** Run: `bun test tests/unit/mdq/query.test.ts -t 'MarkdownDoc chaining'` @@ -629,7 +640,7 @@ describe('selector errors', () => { mdq(doc).query('h2("A") secton("B")'); expect.unreachable(); } catch (error) { - expect(error.index).toBe(7); + expect(error.index).toBe(8); } }); @@ -1238,20 +1249,28 @@ export function removeRanges(source: string, ranges: MatchedRange[]): string { let result = source; for (let i = ordered.length - 1; i >= 0; i--) { const range = ordered[i]; - const end = blockEnd(range); - let start = range.start; - if (!range.trailing) start = trimPrecedingBlankLine(result, start); - result = result.slice(0, start) + result.slice(end); + const head = result.slice(0, range.start); + const tail = result.slice(blockEnd(range)); + if (tail) { + result = head + tail; + continue; + } + if (!head) { + result = ''; + continue; + } + result = `${head.replace(/\n+$/, '')}\n`; } return result; } export function insertAt(source: string, offset: number, markdown: string): string { const block = normalizeBlock(markdown); - const before = source.slice(0, offset); - const after = source.slice(offset); - if (!after) return `${before}${before.endsWith('\n') ? '' : '\n'}\n${block}`.replace(/\n{3,}/g, '\n\n'); - return `${before}${block}\n${after}`.replace(/\n{3,}/g, '\n\n'); + const before = source.slice(0, offset).replace(/\n+$/, ''); + const after = source.slice(offset).replace(/^\n+/, ''); + if (!before) return `${block}\n${after}`; + if (!after) return `${before}\n\n${block}`; + return `${before}\n\n${block}\n${after}`; } function dedupeRanges(ranges: MatchedRange[]): MatchedRange[] { @@ -1266,15 +1285,24 @@ function dedupeRanges(ranges: MatchedRange[]): MatchedRange[] { return kept; } -function trimPrecedingBlankLine(source: string, start: number): number { - let cursor = start; - while (cursor > 0 && source[cursor - 1] === '\n') cursor--; - if (cursor === 0) return 0; - return cursor + 1; -} ``` -Note `insertAt` collapses any run of three or more newlines to exactly two. That single rule is what enforces the invariant across every insert path, rather than each verb reasoning about separators itself. +Both rules below were derived from real `marked` output, not assumed. **Do not "simplify" +either one.** + +**`insertAt` normalizes only the seam.** It strips newlines from the end of `before` and +the start of `after`, then rebuilds the join. The tempting alternative — a global +`.replace(/\n{3,}/g, '\n\n')` over the document — is wrong: a fenced code block's raw really +does contain runs of blank lines (`marked` lexes a ```js block holding `a\n\n\nb` as one +`code` token whose raw carries `\n\n\n`), so a global collapse silently rewrites user code. + +**`removeRanges` collapses only at end-of-document.** `marked` bakes the separator into +some raws and not others: a mid-document `paragraph` raw is `"para"` with a sibling `space` +token, a document-final `paragraph` raw is `"last\n"` with no sibling, and every `heading` +raw carries its own `"\n\n"`. So "no trailing space, therefore trim backwards" is wrong — on +`'# A\n\n## B\n\ntext\n'` it eats a blank line and yields `'# A\ntext\n'`. Deleting +`[start, blockEnd)` is already correct whenever anything follows; only a node removed from +the very end needs repair. - [ ] **Step 4: Wire the verbs onto Selection and MarkdownDoc** @@ -1624,7 +1652,7 @@ import YAML from 'yaml'; export function writeFrontmatter(source: string, key: string, value: unknown): string { const { raw, body, offset } = splitFrontmatter(source); - const document = YAML.parseDocument(raw || ''); + const document = raw ? YAML.parseDocument(raw) : new YAML.Document({}); if (value === null) document.delete(key); if (value !== null) document.set(key, value); const rendered = document.toString().replace(/\s+$/, ''); @@ -1853,7 +1881,9 @@ describe('edits', () => { }); it('sets an entry', async () => { - expect((await runMdq(['blockquote', '--set', 'Container=.x'], '> Container: .old\n')).output).toContain('.x'); + const result = await runMdq(['blockquote', '--set', 'Container=.x'], '> Container: .old\n'); + expect(result.output).toContain('.x'); + expect(result.output).not.toContain('.old'); }); }); From 1f7ca9a236241b899726c30f627d0798bb332268 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:13:44 +0300 Subject: [PATCH 05/21] feat(mdq): frontmatter-aware token index Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- src/utils/mdq/edit.ts | 12 + src/utils/mdq/query.ts | 531 +++++++++++++++++++++++++++++ tests/unit/mdq/frontmatter.test.ts | 62 ++++ 3 files changed, 605 insertions(+) create mode 100644 src/utils/mdq/edit.ts create mode 100644 src/utils/mdq/query.ts create mode 100644 tests/unit/mdq/frontmatter.test.ts diff --git a/src/utils/mdq/edit.ts b/src/utils/mdq/edit.ts new file mode 100644 index 00000000..1f819f38 --- /dev/null +++ b/src/utils/mdq/edit.ts @@ -0,0 +1,12 @@ +export function splitFrontmatter(source: string): FrontmatterSplit { + if (!source.startsWith('---')) return { raw: '', body: source, offset: 0 }; + const match = source.match(/^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/); + if (!match) return { raw: '', body: source, offset: 0 }; + return { raw: match[1], body: source.slice(match[0].length), offset: match[0].length }; +} + +export interface FrontmatterSplit { + raw: string; + body: string; + offset: number; +} diff --git a/src/utils/mdq/query.ts b/src/utils/mdq/query.ts new file mode 100644 index 00000000..8781d5e8 --- /dev/null +++ b/src/utils/mdq/query.ts @@ -0,0 +1,531 @@ +import { type Token, type Tokens, marked } from 'marked'; +import { splitFrontmatter } from './edit.ts'; + +export { splitFrontmatter }; + +export type SelectorType = 'section' | 'section1' | 'section2' | 'section3' | 'section4' | 'section5' | 'section6' | 'table' | 'heading' | 'paragraph' | 'list' | 'item' | 'code' | 'blockquote' | 'hr' | 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6'; + +export interface TextMatcher { + mode: 'exact' | 'contains' | 'regex'; + value: string; + negated: boolean; +} + +export interface QuerySegment { + selector: SelectorType; + textMatch?: TextMatcher; + index: number | null; + slice: { from?: number; to?: number } | null; +} + +export interface MatchedRange { + token: Token; + start: number; + length: number; + trailing?: { start: number; length: number }; + innerTokens?: MatchedRange[]; +} + +export function parseQuery(input: string): QuerySegment[] { + const segments: QuerySegment[] = []; + let pos = 0; + + function peek(): string { + return pos < input.length ? input[pos] : ''; + } + + function advance(): string { + return input[pos++] || ''; + } + + function skipWhitespace() { + while (pos < input.length && /\s/.test(input[pos])) pos++; + } + + function readIdentifier(): string { + const start = pos; + while (pos < input.length && /[a-zA-Z_\d]/.test(input[pos])) pos++; + return input.slice(start, pos); + } + + function readQuotedString(): string { + const quote = advance(); + let result = ''; + while (pos < input.length && input[pos] !== quote) { + if (input[pos] === '\\') { + pos++; + result += input[pos] || ''; + } else { + result += input[pos]; + } + pos++; + } + if (pos < input.length) pos++; + return result; + } + + function readUntilAny(chars: string): string { + const start = pos; + while (pos < input.length && !chars.includes(input[pos])) pos++; + return input.slice(start, pos); + } + + function parseTextMatcher(): TextMatcher { + let negated = false; + if (peek() === '!') { + negated = true; + advance(); + } + + if (peek() === '~') { + advance(); + const value = readQuotedString(); + return { mode: 'contains', value, negated }; + } + + if (peek() === '/') { + advance(); + let value = ''; + while (pos < input.length && input[pos] !== '/') { + value += input[pos]; + pos++; + } + if (pos < input.length) pos++; + while (pos < input.length && /[gimsuy]/.test(input[pos])) pos++; + return { mode: 'regex', value, negated }; + } + + const value = readQuotedString(); + return { mode: 'exact', value, negated }; + } + + while (pos < input.length) { + skipWhitespace(); + if (pos >= input.length) break; + + const selector = readIdentifier(); + if (!selector) { + pos++; + continue; + } + + const segment: QuerySegment = { + selector: selector as SelectorType, + index: null, + slice: null, + }; + + if (peek() === '(') { + advance(); + segment.textMatch = parseTextMatcher(); + if (peek() === ')') advance(); + } + + while (peek() === '[') { + advance(); + const content = readUntilAny(']'); + if (/^-?\d*(:-?\d*)?$/.test(content) && content !== '') { + if (content.includes(':')) { + const colonIdx = content.indexOf(':'); + const fromStr = content.slice(0, colonIdx); + const toStr = content.slice(colonIdx + 1); + segment.slice = { + from: fromStr ? Number.parseInt(fromStr, 10) : undefined, + to: toStr ? Number.parseInt(toStr, 10) : undefined, + }; + } else { + segment.index = Number.parseInt(content, 10); + } + } + if (peek() === ']') advance(); + } + + segments.push(segment); + } + + return segments; +} + +function matchText(text: string, matcher: TextMatcher): boolean { + let result: boolean; + + switch (matcher.mode) { + case 'exact': + result = text === matcher.value; + break; + case 'contains': + result = text.includes(matcher.value); + break; + case 'regex': + result = new RegExp(matcher.value, 'i').test(text); + break; + default: + result = false; + } + + return matcher.negated ? !result : result; +} + +function entryKey(line: string): string | null { + const separator = line.indexOf(':'); + if (separator < 1) return null; + return line.slice(0, separator).trim().toLowerCase(); +} + +function getTokenText(token: Token): string { + const t = token as any; + switch (token.type) { + case 'heading': + case 'paragraph': + case 'code': + case 'blockquote': + case 'list_item': + return t.text || ''; + case 'table': + return (t.header || []).map((h: any) => h.text).join(', '); + default: + return ''; + } +} + +function getHeadingDepth(selector: string): number | null { + const match = selector.match(/^h([1-6])$/); + return match ? Number.parseInt(match[1], 10) : null; +} + +function isSectionSelector(selector: string): boolean { + return /^section\d?$/.test(selector); +} + +function getSectionDepth(selector: string): number | null { + const match = selector.match(/^section([1-6])$/); + return match ? Number.parseInt(match[1], 10) : null; +} + +function selectorToTokenType(selector: string): string | null { + if (/^h[1-6]$/.test(selector)) return 'heading'; + const map: Record = { + heading: 'heading', + paragraph: 'paragraph', + table: 'table', + code: 'code', + list: 'list', + blockquote: 'blockquote', + hr: 'hr', + item: 'list_item', + }; + return map[selector] || null; +} + +export function buildTokenIndex(source: string): MatchedRange[] { + const { body, offset } = splitFrontmatter(source); + const tokens = marked.lexer(body); + const ranges: MatchedRange[] = []; + let cursor = offset; + + for (const token of tokens) { + const raw = (token as any).raw || ''; + if (token.type === 'space') { + const previous = ranges[ranges.length - 1]; + if (previous) previous.trailing = { start: cursor, length: raw.length }; + cursor += raw.length; + continue; + } + ranges.push({ token, start: cursor, length: raw.length }); + cursor += raw.length; + } + + return ranges; +} + +function computeSections(candidates: MatchedRange[], segment: QuerySegment): MatchedRange[] { + const sectionDepth = getSectionDepth(segment.selector as string); + const sections: MatchedRange[] = []; + + for (let i = 0; i < candidates.length; i++) { + const range = candidates[i]; + if (range.token.type !== 'heading') continue; + + const heading = range.token as Tokens.Heading; + if (sectionDepth !== null && heading.depth !== sectionDepth) continue; + if (segment.textMatch && !matchText(heading.text, segment.textMatch)) continue; + + const depth = heading.depth; + const innerTokens: MatchedRange[] = []; + let endOffset = range.start + range.length; + + for (let j = i + 1; j < candidates.length; j++) { + const nextRange = candidates[j]; + if (nextRange.token.type === 'heading' && (nextRange.token as Tokens.Heading).depth <= depth) break; + innerTokens.push(nextRange); + endOffset = nextRange.start + nextRange.length; + if (nextRange.trailing) endOffset = nextRange.trailing.start + nextRange.trailing.length; + } + + sections.push({ + token: range.token, + start: range.start, + length: endOffset - range.start, + innerTokens, + }); + } + + return sections; +} + +function extractListItems(candidates: MatchedRange[]): MatchedRange[] { + const items: MatchedRange[] = []; + + for (const range of candidates) { + if (range.token.type !== 'list') continue; + + const list = range.token as Tokens.List; + const listRaw = (range.token as any).raw as string; + let searchFrom = 0; + + for (const item of list.items) { + const itemRaw = (item as any).raw as string; + const idx = listRaw.indexOf(itemRaw, searchFrom); + if (idx === -1) continue; + + items.push({ + token: item as unknown as Token, + start: range.start + idx, + length: itemRaw.length, + }); + + searchFrom = idx + itemRaw.length; + } + } + + return items; +} + +function applyIndexSlice(matches: MatchedRange[], segment: QuerySegment): MatchedRange[] { + if (segment.index !== null) { + const idx = segment.index < 0 ? matches.length + segment.index : segment.index; + return idx >= 0 && idx < matches.length ? [matches[idx]] : []; + } + + if (segment.slice) { + const { from, to } = segment.slice; + return matches.slice(from, to); + } + + return matches; +} + +function expandSectionRanges(matches: MatchedRange[]): MatchedRange[] { + let hasSection = false; + for (const m of matches) { + if (m.innerTokens) { + hasSection = true; + break; + } + } + if (!hasSection) return matches; + + const expanded: MatchedRange[] = []; + for (const m of matches) { + if (m.innerTokens) { + expanded.push({ token: m.token, start: m.start, length: ((m.token as any).raw || '').length }); + expanded.push(...m.innerTokens); + } else { + expanded.push(m); + } + } + return expanded; +} + +function executeSegments(candidates: MatchedRange[], segments: QuerySegment[]): MatchedRange[] { + if (segments.length === 0) return candidates; + + const segment = segments[0]; + const remaining = segments.slice(1); + + if (isSectionSelector(segment.selector as string)) { + const sections = computeSections(candidates, segment); + const indexed = applyIndexSlice(sections, segment); + + if (remaining.length === 0) return indexed; + + const results: MatchedRange[] = []; + for (const section of indexed) { + results.push(...executeSegments(section.innerTokens || [], remaining)); + } + return results; + } + + if (segment.selector === 'item') { + let items = extractListItems(candidates); + if (segment.textMatch) { + items = items.filter((r) => matchText(getTokenText(r.token), segment.textMatch!)); + } + return executeSegments(applyIndexSlice(items, segment), remaining); + } + + const tokenType = selectorToTokenType(segment.selector as string); + if (!tokenType) return []; + + let matches = candidates.filter((r) => r.token.type === tokenType); + + const depth = getHeadingDepth(segment.selector as string); + if (depth !== null) { + matches = matches.filter((r) => (r.token as any).depth === depth); + } + + if (segment.textMatch) { + matches = matches.filter((r) => matchText(getTokenText(r.token), segment.textMatch!)); + } + + return executeSegments(applyIndexSlice(matches, segment), remaining); +} + +export class MarkdownQuery { + private source: string; + private matches: MatchedRange[]; + + constructor(source: string, matches?: MatchedRange[]) { + this.source = source; + this.matches = matches || buildTokenIndex(source); + } + + query(selector: string): MarkdownQuery { + const segments = parseQuery(selector); + const candidates = expandSectionRanges(this.matches); + const results = executeSegments(candidates, segments); + return new MarkdownQuery(this.source, results); + } + + text(): string { + return this.matches.map((r) => this.source.slice(r.start, r.start + r.length)).join(''); + } + + get(): string { + return this.text(); + } + + toJson(): Record[] { + const results: Record[] = []; + + for (const range of this.matches) { + if (range.token.type !== 'table') continue; + + const table = range.token as Tokens.Table; + const headers = table.header.map((h) => h.text); + for (const row of table.rows) { + const obj: Record = {}; + for (let i = 0; i < headers.length; i++) { + obj[headers[i]] = row[i]?.text ?? ''; + } + results.push(obj); + } + } + + return results; + } + + keyValue(): Record { + const entries: Record = {}; + + for (const range of this.matches) { + for (const line of getTokenText(range.token).split('\n')) { + const key = entryKey(line); + if (!key) continue; + const value = line.slice(line.indexOf(':') + 1).trim(); + if (value) entries[key] = value; + } + } + + return entries; + } + + setKeyValue(key: string, value: string | null): string { + return this.replaceEach((match) => { + const token = match.matches[0].token; + const lines = getTokenText(token) + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); + + const index = lines.findIndex((line) => entryKey(line) === key.toLowerCase()); + if (index < 0 && value) lines.push(`${key}: ${value}`); + if (index >= 0 && value) lines[index] = `${key}: ${value}`; + if (index >= 0 && !value) lines.splice(index, 1); + + if (token.type !== 'blockquote') return lines.join('\n'); + return lines.map((line) => `> ${line}`).join('\n'); + }); + } + + replace(content: string): string { + return this.replaceEach(() => content); + } + + replaceEach(replacer: (match: MarkdownQuery, index: number) => string): string { + const sorted = [...this.matches].sort((a, b) => a.start - b.start); + + const kept: MatchedRange[] = []; + let lastEnd = -1; + for (const range of sorted) { + if (range.start < lastEnd) continue; + kept.push(range); + lastEnd = range.start + range.length; + } + + const replacements = kept.map((range, index) => replacer(new MarkdownQuery(this.source, [range]), index)); + let result = this.source; + for (let i = kept.length - 1; i >= 0; i--) { + const range = kept[i]; + result = result.slice(0, range.start) + replacements[i] + result.slice(range.start + range.length); + } + + return result; + } + + count(): number { + return this.matches.length; + } + + first(): MarkdownQuery { + return new MarkdownQuery(this.source, this.matches.slice(0, 1)); + } + + last(): MarkdownQuery { + return new MarkdownQuery(this.source, this.matches.slice(-1)); + } + + before(): MarkdownQuery { + if (this.matches.length === 0) return new MarkdownQuery(this.source, []); + const cutoff = this.matches[0].start; + const allTokens = buildTokenIndex(this.source); + const beforeTokens = allTokens.filter((r) => r.start + r.length <= cutoff); + return new MarkdownQuery(this.source, beforeTokens); + } + + after(): MarkdownQuery { + if (this.matches.length === 0) return new MarkdownQuery(this.source, []); + const lastMatch = this.matches[this.matches.length - 1]; + const cutoff = lastMatch.start + lastMatch.length; + const allTokens = buildTokenIndex(this.source); + const afterTokens = allTokens.filter((r) => r.start >= cutoff); + return new MarkdownQuery(this.source, afterTokens); + } + + each(): MarkdownQuery[] { + return this.matches.map((m) => new MarkdownQuery(this.source, [m])); + } + + meta(): Array<{ type: string; depth: number | null; text: string }> { + return this.matches.map((range) => { + const token = range.token as any; + let depth: number | null = null; + if (token.type === 'heading') depth = token.depth; + return { type: token.type, depth, text: getTokenText(range.token) }; + }); + } +} + +export function mdq(source: string): MarkdownQuery { + return new MarkdownQuery(source); +} diff --git a/tests/unit/mdq/frontmatter.test.ts b/tests/unit/mdq/frontmatter.test.ts new file mode 100644 index 00000000..ccace300 --- /dev/null +++ b/tests/unit/mdq/frontmatter.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'vitest'; +import { splitFrontmatter } from '../../../src/utils/mdq/edit.ts'; +import { buildTokenIndex } from '../../../src/utils/mdq/query.ts'; + +describe('splitFrontmatter', () => { + it('splits a leading yaml block from the body', () => { + const src = '---\nurl: /login\nwait: 1000\n---\n\n# Title\n'; + const fm = splitFrontmatter(src); + expect(fm.raw).toBe('url: /login\nwait: 1000'); + expect(fm.body).toBe('\n# Title\n'); + expect(fm.offset).toBe(src.length - fm.body.length); + }); + + it('returns no frontmatter when the document does not open with ---', () => { + const fm = splitFrontmatter('# Title\n\n---\n'); + expect(fm.raw).toBe(''); + expect(fm.offset).toBe(0); + }); + + it('treats an unterminated --- as body, not frontmatter', () => { + const fm = splitFrontmatter('---\nnot closed\n'); + expect(fm.raw).toBe(''); + expect(fm.offset).toBe(0); + }); +}); + +describe('buildTokenIndex', () => { + it('excludes frontmatter so it is never lexed as a setext heading', () => { + const ranges = buildTokenIndex('---\nurl: /login\n---\n\n# Title\n'); + expect(ranges.filter((r) => r.token.type === 'heading')).toHaveLength(1); + expect(ranges.every((r) => r.start >= 20)).toBe(true); + }); + + it('keeps offsets absolute so slicing the original source works', () => { + const src = '---\nurl: /x\n---\n\n# Title\n'; + const ranges = buildTokenIndex(src); + const heading = ranges.find((r) => r.token.type === 'heading'); + expect(src.slice(heading!.start, heading!.start + heading!.length)).toBe('# Title\n'); + }); + + it('records the trailing space token of a paragraph', () => { + const ranges = buildTokenIndex('para\n\n# Next\n'); + const para = ranges.find((r) => r.token.type === 'paragraph'); + expect(para!.length).toBe(4); + expect(para!.trailing).toEqual({ start: 4, length: 2 }); + }); + + it('leaves trailing undefined for a node with no following space', () => { + expect(buildTokenIndex('# Only\n')[0].trailing).toBeUndefined(); + }); + + it('bakes the separator into a document-final paragraph instead of a space token', () => { + const ranges = buildTokenIndex('# A\n\nlast\n'); + const para = ranges.find((r) => r.token.type === 'paragraph'); + expect(para!.length).toBe(5); + expect(para!.trailing).toBeUndefined(); + }); + + it('never yields a space token as a match', () => { + expect(buildTokenIndex('a\n\nb\n\nc\n').some((r) => r.token.type === 'space')).toBe(false); + }); +}); From bcfd076b31682ac0c1f35fca3b6e030250f81cfa Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:14:49 +0300 Subject: [PATCH 06/21] refactor(mdq): move markdown-query into src/utils/mdq behind a shim Types moved to end of file and ternaries replaced with early returns, per repo style. All 1416 unit tests pass through the shim. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- src/utils/markdown-query.ts | 520 +----------------- src/utils/mdq/query.ts | 73 +-- .../query.test.ts} | 2 +- 3 files changed, 44 insertions(+), 551 deletions(-) rename tests/unit/{markdown-query.test.ts => mdq/query.test.ts} (99%) diff --git a/src/utils/markdown-query.ts b/src/utils/markdown-query.ts index 0930a27f..423d4635 100644 --- a/src/utils/markdown-query.ts +++ b/src/utils/markdown-query.ts @@ -1,519 +1 @@ -import { type Token, type Tokens, marked } from 'marked'; - -export type SelectorType = 'section' | 'section1' | 'section2' | 'section3' | 'section4' | 'section5' | 'section6' | 'table' | 'heading' | 'paragraph' | 'list' | 'item' | 'code' | 'blockquote' | 'hr' | 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6'; - -export interface TextMatcher { - mode: 'exact' | 'contains' | 'regex'; - value: string; - negated: boolean; -} - -export interface QuerySegment { - selector: SelectorType; - textMatch?: TextMatcher; - index: number | null; - slice: { from?: number; to?: number } | null; -} - -export interface MatchedRange { - token: Token; - start: number; - length: number; - innerTokens?: MatchedRange[]; -} - -export function parseQuery(input: string): QuerySegment[] { - const segments: QuerySegment[] = []; - let pos = 0; - - function peek(): string { - return pos < input.length ? input[pos] : ''; - } - - function advance(): string { - return input[pos++] || ''; - } - - function skipWhitespace() { - while (pos < input.length && /\s/.test(input[pos])) pos++; - } - - function readIdentifier(): string { - const start = pos; - while (pos < input.length && /[a-zA-Z_\d]/.test(input[pos])) pos++; - return input.slice(start, pos); - } - - function readQuotedString(): string { - const quote = advance(); - let result = ''; - while (pos < input.length && input[pos] !== quote) { - if (input[pos] === '\\') { - pos++; - result += input[pos] || ''; - } else { - result += input[pos]; - } - pos++; - } - if (pos < input.length) pos++; - return result; - } - - function readUntilAny(chars: string): string { - const start = pos; - while (pos < input.length && !chars.includes(input[pos])) pos++; - return input.slice(start, pos); - } - - function parseTextMatcher(): TextMatcher { - let negated = false; - if (peek() === '!') { - negated = true; - advance(); - } - - if (peek() === '~') { - advance(); - const value = readQuotedString(); - return { mode: 'contains', value, negated }; - } - - if (peek() === '/') { - advance(); - let value = ''; - while (pos < input.length && input[pos] !== '/') { - value += input[pos]; - pos++; - } - if (pos < input.length) pos++; - while (pos < input.length && /[gimsuy]/.test(input[pos])) pos++; - return { mode: 'regex', value, negated }; - } - - const value = readQuotedString(); - return { mode: 'exact', value, negated }; - } - - while (pos < input.length) { - skipWhitespace(); - if (pos >= input.length) break; - - const selector = readIdentifier(); - if (!selector) { - pos++; - continue; - } - - const segment: QuerySegment = { - selector: selector as SelectorType, - index: null, - slice: null, - }; - - if (peek() === '(') { - advance(); - segment.textMatch = parseTextMatcher(); - if (peek() === ')') advance(); - } - - while (peek() === '[') { - advance(); - const content = readUntilAny(']'); - if (/^-?\d*(:-?\d*)?$/.test(content) && content !== '') { - if (content.includes(':')) { - const colonIdx = content.indexOf(':'); - const fromStr = content.slice(0, colonIdx); - const toStr = content.slice(colonIdx + 1); - segment.slice = { - from: fromStr ? Number.parseInt(fromStr, 10) : undefined, - to: toStr ? Number.parseInt(toStr, 10) : undefined, - }; - } else { - segment.index = Number.parseInt(content, 10); - } - } - if (peek() === ']') advance(); - } - - segments.push(segment); - } - - return segments; -} - -function matchText(text: string, matcher: TextMatcher): boolean { - let result: boolean; - - switch (matcher.mode) { - case 'exact': - result = text === matcher.value; - break; - case 'contains': - result = text.includes(matcher.value); - break; - case 'regex': - result = new RegExp(matcher.value, 'i').test(text); - break; - default: - result = false; - } - - return matcher.negated ? !result : result; -} - -function entryKey(line: string): string | null { - const separator = line.indexOf(':'); - if (separator < 1) return null; - return line.slice(0, separator).trim().toLowerCase(); -} - -function getTokenText(token: Token): string { - const t = token as any; - switch (token.type) { - case 'heading': - case 'paragraph': - case 'code': - case 'blockquote': - case 'list_item': - return t.text || ''; - case 'table': - return (t.header || []).map((h: any) => h.text).join(', '); - default: - return ''; - } -} - -function getHeadingDepth(selector: string): number | null { - const match = selector.match(/^h([1-6])$/); - return match ? Number.parseInt(match[1], 10) : null; -} - -function isSectionSelector(selector: string): boolean { - return /^section\d?$/.test(selector); -} - -function getSectionDepth(selector: string): number | null { - const match = selector.match(/^section([1-6])$/); - return match ? Number.parseInt(match[1], 10) : null; -} - -function selectorToTokenType(selector: string): string | null { - if (/^h[1-6]$/.test(selector)) return 'heading'; - const map: Record = { - heading: 'heading', - paragraph: 'paragraph', - table: 'table', - code: 'code', - list: 'list', - blockquote: 'blockquote', - hr: 'hr', - item: 'list_item', - }; - return map[selector] || null; -} - -function buildTokenIndex(source: string): MatchedRange[] { - const tokens = marked.lexer(source); - const ranges: MatchedRange[] = []; - let offset = 0; - - for (const token of tokens) { - const raw = (token as any).raw || ''; - ranges.push({ token, start: offset, length: raw.length }); - offset += raw.length; - } - - return ranges; -} - -function computeSections(candidates: MatchedRange[], segment: QuerySegment): MatchedRange[] { - const sectionDepth = getSectionDepth(segment.selector as string); - const sections: MatchedRange[] = []; - - for (let i = 0; i < candidates.length; i++) { - const range = candidates[i]; - if (range.token.type !== 'heading') continue; - - const heading = range.token as Tokens.Heading; - if (sectionDepth !== null && heading.depth !== sectionDepth) continue; - if (segment.textMatch && !matchText(heading.text, segment.textMatch)) continue; - - const depth = heading.depth; - const innerTokens: MatchedRange[] = []; - let endOffset = range.start + range.length; - - for (let j = i + 1; j < candidates.length; j++) { - const nextRange = candidates[j]; - if (nextRange.token.type === 'heading' && (nextRange.token as Tokens.Heading).depth <= depth) break; - innerTokens.push(nextRange); - endOffset = nextRange.start + nextRange.length; - } - - sections.push({ - token: range.token, - start: range.start, - length: endOffset - range.start, - innerTokens, - }); - } - - return sections; -} - -function extractListItems(candidates: MatchedRange[]): MatchedRange[] { - const items: MatchedRange[] = []; - - for (const range of candidates) { - if (range.token.type !== 'list') continue; - - const list = range.token as Tokens.List; - const listRaw = (range.token as any).raw as string; - let searchFrom = 0; - - for (const item of list.items) { - const itemRaw = (item as any).raw as string; - const idx = listRaw.indexOf(itemRaw, searchFrom); - if (idx === -1) continue; - - items.push({ - token: item as unknown as Token, - start: range.start + idx, - length: itemRaw.length, - }); - - searchFrom = idx + itemRaw.length; - } - } - - return items; -} - -function applyIndexSlice(matches: MatchedRange[], segment: QuerySegment): MatchedRange[] { - if (segment.index !== null) { - const idx = segment.index < 0 ? matches.length + segment.index : segment.index; - return idx >= 0 && idx < matches.length ? [matches[idx]] : []; - } - - if (segment.slice) { - const { from, to } = segment.slice; - return matches.slice(from, to); - } - - return matches; -} - -function expandSectionRanges(matches: MatchedRange[]): MatchedRange[] { - let hasSection = false; - for (const m of matches) { - if (m.innerTokens) { - hasSection = true; - break; - } - } - if (!hasSection) return matches; - - const expanded: MatchedRange[] = []; - for (const m of matches) { - if (m.innerTokens) { - expanded.push({ token: m.token, start: m.start, length: ((m.token as any).raw || '').length }); - expanded.push(...m.innerTokens); - } else { - expanded.push(m); - } - } - return expanded; -} - -function executeSegments(candidates: MatchedRange[], segments: QuerySegment[]): MatchedRange[] { - if (segments.length === 0) return candidates; - - const segment = segments[0]; - const remaining = segments.slice(1); - - if (isSectionSelector(segment.selector as string)) { - const sections = computeSections(candidates, segment); - const indexed = applyIndexSlice(sections, segment); - - if (remaining.length === 0) return indexed; - - const results: MatchedRange[] = []; - for (const section of indexed) { - results.push(...executeSegments(section.innerTokens || [], remaining)); - } - return results; - } - - if (segment.selector === 'item') { - let items = extractListItems(candidates); - if (segment.textMatch) { - items = items.filter((r) => matchText(getTokenText(r.token), segment.textMatch!)); - } - return executeSegments(applyIndexSlice(items, segment), remaining); - } - - const tokenType = selectorToTokenType(segment.selector as string); - if (!tokenType) return []; - - let matches = candidates.filter((r) => r.token.type === tokenType); - - const depth = getHeadingDepth(segment.selector as string); - if (depth !== null) { - matches = matches.filter((r) => (r.token as any).depth === depth); - } - - if (segment.textMatch) { - matches = matches.filter((r) => matchText(getTokenText(r.token), segment.textMatch!)); - } - - return executeSegments(applyIndexSlice(matches, segment), remaining); -} - -export class MarkdownQuery { - private source: string; - private matches: MatchedRange[]; - - constructor(source: string, matches?: MatchedRange[]) { - this.source = source; - this.matches = matches || buildTokenIndex(source); - } - - query(selector: string): MarkdownQuery { - const segments = parseQuery(selector); - const candidates = expandSectionRanges(this.matches); - const results = executeSegments(candidates, segments); - return new MarkdownQuery(this.source, results); - } - - text(): string { - return this.matches.map((r) => this.source.slice(r.start, r.start + r.length)).join(''); - } - - get(): string { - return this.text(); - } - - toJson(): Record[] { - const results: Record[] = []; - - for (const range of this.matches) { - if (range.token.type !== 'table') continue; - - const table = range.token as Tokens.Table; - const headers = table.header.map((h) => h.text); - for (const row of table.rows) { - const obj: Record = {}; - for (let i = 0; i < headers.length; i++) { - obj[headers[i]] = row[i]?.text ?? ''; - } - results.push(obj); - } - } - - return results; - } - - keyValue(): Record { - const entries: Record = {}; - - for (const range of this.matches) { - for (const line of getTokenText(range.token).split('\n')) { - const key = entryKey(line); - if (!key) continue; - const value = line.slice(line.indexOf(':') + 1).trim(); - if (value) entries[key] = value; - } - } - - return entries; - } - - setKeyValue(key: string, value: string | null): string { - return this.replaceEach((match) => { - const token = match.matches[0].token; - const lines = getTokenText(token) - .split('\n') - .map((line) => line.trim()) - .filter(Boolean); - - const index = lines.findIndex((line) => entryKey(line) === key.toLowerCase()); - if (index < 0 && value) lines.push(`${key}: ${value}`); - if (index >= 0 && value) lines[index] = `${key}: ${value}`; - if (index >= 0 && !value) lines.splice(index, 1); - - if (token.type !== 'blockquote') return lines.join('\n'); - return lines.map((line) => `> ${line}`).join('\n'); - }); - } - - replace(content: string): string { - return this.replaceEach(() => content); - } - - replaceEach(replacer: (match: MarkdownQuery, index: number) => string): string { - const sorted = [...this.matches].sort((a, b) => a.start - b.start); - - const kept: MatchedRange[] = []; - let lastEnd = -1; - for (const range of sorted) { - if (range.start < lastEnd) continue; - kept.push(range); - lastEnd = range.start + range.length; - } - - const replacements = kept.map((range, index) => replacer(new MarkdownQuery(this.source, [range]), index)); - let result = this.source; - for (let i = kept.length - 1; i >= 0; i--) { - const range = kept[i]; - result = result.slice(0, range.start) + replacements[i] + result.slice(range.start + range.length); - } - - return result; - } - - count(): number { - return this.matches.length; - } - - first(): MarkdownQuery { - return new MarkdownQuery(this.source, this.matches.slice(0, 1)); - } - - last(): MarkdownQuery { - return new MarkdownQuery(this.source, this.matches.slice(-1)); - } - - before(): MarkdownQuery { - if (this.matches.length === 0) return new MarkdownQuery(this.source, []); - const cutoff = this.matches[0].start; - const allTokens = buildTokenIndex(this.source); - const beforeTokens = allTokens.filter((r) => r.start + r.length <= cutoff); - return new MarkdownQuery(this.source, beforeTokens); - } - - after(): MarkdownQuery { - if (this.matches.length === 0) return new MarkdownQuery(this.source, []); - const lastMatch = this.matches[this.matches.length - 1]; - const cutoff = lastMatch.start + lastMatch.length; - const allTokens = buildTokenIndex(this.source); - const afterTokens = allTokens.filter((r) => r.start >= cutoff); - return new MarkdownQuery(this.source, afterTokens); - } - - each(): MarkdownQuery[] { - return this.matches.map((m) => new MarkdownQuery(this.source, [m])); - } - - meta(): Array<{ type: string; depth: number | null; text: string }> { - return this.matches.map((range) => { - const token = range.token as any; - let depth: number | null = null; - if (token.type === 'heading') depth = token.depth; - return { type: token.type, depth, text: getTokenText(range.token) }; - }); - } -} - -export function mdq(source: string): MarkdownQuery { - return new MarkdownQuery(source); -} +export * from './mdq/query.ts'; diff --git a/src/utils/mdq/query.ts b/src/utils/mdq/query.ts index 8781d5e8..a1f9e404 100644 --- a/src/utils/mdq/query.ts +++ b/src/utils/mdq/query.ts @@ -3,35 +3,13 @@ import { splitFrontmatter } from './edit.ts'; export { splitFrontmatter }; -export type SelectorType = 'section' | 'section1' | 'section2' | 'section3' | 'section4' | 'section5' | 'section6' | 'table' | 'heading' | 'paragraph' | 'list' | 'item' | 'code' | 'blockquote' | 'hr' | 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6'; - -export interface TextMatcher { - mode: 'exact' | 'contains' | 'regex'; - value: string; - negated: boolean; -} - -export interface QuerySegment { - selector: SelectorType; - textMatch?: TextMatcher; - index: number | null; - slice: { from?: number; to?: number } | null; -} - -export interface MatchedRange { - token: Token; - start: number; - length: number; - trailing?: { start: number; length: number }; - innerTokens?: MatchedRange[]; -} - export function parseQuery(input: string): QuerySegment[] { const segments: QuerySegment[] = []; let pos = 0; function peek(): string { - return pos < input.length ? input[pos] : ''; + if (pos >= input.length) return ''; + return input[pos]; } function advance(): string { @@ -130,8 +108,8 @@ export function parseQuery(input: string): QuerySegment[] { const fromStr = content.slice(0, colonIdx); const toStr = content.slice(colonIdx + 1); segment.slice = { - from: fromStr ? Number.parseInt(fromStr, 10) : undefined, - to: toStr ? Number.parseInt(toStr, 10) : undefined, + from: parseBound(fromStr), + to: parseBound(toStr), }; } else { segment.index = Number.parseInt(content, 10); @@ -146,6 +124,11 @@ export function parseQuery(input: string): QuerySegment[] { return segments; } +function parseBound(value: string): number | undefined { + if (!value) return undefined; + return Number.parseInt(value, 10); +} + function matchText(text: string, matcher: TextMatcher): boolean { let result: boolean; @@ -163,7 +146,8 @@ function matchText(text: string, matcher: TextMatcher): boolean { result = false; } - return matcher.negated ? !result : result; + if (matcher.negated) return !result; + return result; } function entryKey(line: string): string | null { @@ -190,7 +174,8 @@ function getTokenText(token: Token): string { function getHeadingDepth(selector: string): number | null { const match = selector.match(/^h([1-6])$/); - return match ? Number.parseInt(match[1], 10) : null; + if (!match) return null; + return Number.parseInt(match[1], 10); } function isSectionSelector(selector: string): boolean { @@ -199,7 +184,8 @@ function isSectionSelector(selector: string): boolean { function getSectionDepth(selector: string): number | null { const match = selector.match(/^section([1-6])$/); - return match ? Number.parseInt(match[1], 10) : null; + if (!match) return null; + return Number.parseInt(match[1], 10); } function selectorToTokenType(selector: string): string | null { @@ -303,8 +289,10 @@ function extractListItems(candidates: MatchedRange[]): MatchedRange[] { function applyIndexSlice(matches: MatchedRange[], segment: QuerySegment): MatchedRange[] { if (segment.index !== null) { - const idx = segment.index < 0 ? matches.length + segment.index : segment.index; - return idx >= 0 && idx < matches.length ? [matches[idx]] : []; + let idx = segment.index; + if (idx < 0) idx = matches.length + idx; + if (idx < 0 || idx >= matches.length) return []; + return [matches[idx]]; } if (segment.slice) { @@ -529,3 +517,26 @@ export class MarkdownQuery { export function mdq(source: string): MarkdownQuery { return new MarkdownQuery(source); } + +export type SelectorType = 'section' | 'section1' | 'section2' | 'section3' | 'section4' | 'section5' | 'section6' | 'table' | 'heading' | 'paragraph' | 'list' | 'item' | 'code' | 'blockquote' | 'hr' | 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6'; + +export interface TextMatcher { + mode: 'exact' | 'contains' | 'regex'; + value: string; + negated: boolean; +} + +export interface QuerySegment { + selector: SelectorType; + textMatch?: TextMatcher; + index: number | null; + slice: { from?: number; to?: number } | null; +} + +export interface MatchedRange { + token: Token; + start: number; + length: number; + trailing?: { start: number; length: number }; + innerTokens?: MatchedRange[]; +} diff --git a/tests/unit/markdown-query.test.ts b/tests/unit/mdq/query.test.ts similarity index 99% rename from tests/unit/markdown-query.test.ts rename to tests/unit/mdq/query.test.ts index f71c30ec..f44d2595 100644 --- a/tests/unit/markdown-query.test.ts +++ b/tests/unit/mdq/query.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { mdq, parseQuery } from '../../src/utils/markdown-query.ts'; +import { mdq, parseQuery } from '../../../src/utils/mdq/query.ts'; const sampleMarkdown = `# Main Title From 2dbae8aa1b46f230b4c92c1fb5435f374438c82a Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:21:13 +0300 Subject: [PATCH 07/21] feat(mdq): writes return MarkdownDoc so edits chain Splits MarkdownQuery into MarkdownDoc (whole document) and Selection (matched ranges, extends MarkdownDoc). Every write returns a MarkdownDoc, so several edits compose in one expression. Migrates 17 call sites, not the 11 the plan predicted. The extras were found by type-checking on the type name repo-wide rather than by file path: a test helper fed a MarkdownDoc to marked.lexer, which does not coerce and threw at runtime. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- boat/prima/src/prima.ts | 3 +- .../plans/2026-09-15-mdq-package.md | 20 ++- src/ai/planner.ts | 15 +- src/ai/researcher.ts | 2 +- src/ai/researcher/deep-analysis.ts | 4 +- src/ai/researcher/locators.ts | 4 +- src/ai/researcher/pagination.ts | 2 +- src/ai/researcher/research-result.ts | 4 +- src/experience-tracker.ts | 7 +- src/utils/mdq/edit.ts | 14 ++ src/utils/mdq/query.ts | 154 ++++++++++++------ tests/unit/mdq/query.test.ts | 50 ++++-- tests/unit/research-parser-pagination.test.ts | 2 +- 13 files changed, 198 insertions(+), 83 deletions(-) diff --git a/boat/prima/src/prima.ts b/boat/prima/src/prima.ts index bb7a06e8..fc489d16 100644 --- a/boat/prima/src/prima.ts +++ b/boat/prima/src/prima.ts @@ -57,7 +57,8 @@ function dropVolatileColumns(markdown: string): string { const divider = `|${columns.map(() => '------').join('|')}|`; const body = rows.map((row) => `| ${columns.map((name) => row[name] || '-').join(' | ')} |`); return [header, divider, ...body, ''].join('\n'); - }); + }) + .toString(); } function cap(text: string, max: number): string { diff --git a/docs/superpowers/plans/2026-09-15-mdq-package.md b/docs/superpowers/plans/2026-09-15-mdq-package.md index a826755b..ac7aed23 100644 --- a/docs/superpowers/plans/2026-09-15-mdq-package.md +++ b/docs/superpowers/plans/2026-09-15-mdq-package.md @@ -511,18 +511,24 @@ Expected: PASS — 114 tests - [ ] **Step 8: Verify with the scoped type check** +Grep by **type name, repo-wide** — not by file path. A path-scoped grep misses test +helpers and bin scripts, and those break too: + ```bash -bunx tsc -p tsconfig.json --noEmit 2>&1 | grep -E "^(src/utils/mdq/|src/utils/markdown-query|src/experience-tracker|src/ai/planner|src/ai/researcher)" +bunx tsc -p tsconfig.json --noEmit 2>&1 | grep -E "MarkdownDoc|MarkdownQuery|Selection" ``` -Expected: **exactly these two lines and nothing else.** +Expected: **no output at all.** -``` -src/ai/researcher/locators.ts(247,41): error TS2339: Property 'playwrightLocatorCount' does not exist on type 'Explorer'. -src/ai/researcher/locators.ts(247,65): error TS7006: Parameter 'page' implicitly has an 'any' type. -``` +Any line is an unmigrated call site. CI will not catch it, because CI runs `tsc --noCheck`. -Any third line is an unmigrated call site. Fix it before continuing — CI will not catch it, because CI runs `tsc --noCheck`. +The audit in Step 6 lists eleven sites; the real count is **seventeen**. The extras are all +the same shape — a function whose declared return type is `string` now returns a +`MarkdownDoc` — in `boat/prima/src/prima.ts`, `planner.ts` (three chains), +`deep-analysis.ts:542` and `experience-tracker.ts:511`, plus a test helper in +`tests/unit/research-parser-pagination.test.ts` that feeds the result to `marked.lexer`, +which does **not** coerce and throws `e.replace is not a function` at runtime. Trust the +grep, not the list. - [ ] **Step 9: Run the full unit suite** diff --git a/src/ai/planner.ts b/src/ai/planner.ts index 0f5162f4..f50bb843 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -301,13 +301,14 @@ export class Planner extends PlannerBase implements Agent { .replaceEach((section) => { const heading = section.query('heading').text().trim(); const withoutHeadings = mdq(section.text()).query('heading').replace(''); - const body = mdq(withoutHeadings).query('hr').replace('').trim(); + const body = mdq(withoutHeadings).query('hr').replace('').toString().trim(); if (body && !seenTitles.has(heading)) { seenTitles.add(heading); return section.text(); } return ''; - }); + }) + .toString(); } const trimmedTitles = new Set(); @@ -319,10 +320,11 @@ export class Planner extends PlannerBase implements Agent { if (trimmedTitles.has(heading)) return section.text(); const count = section.query('blockquote').count(); if (count <= 10) return section.text(); - const kept = mdq(section.text()).query('blockquote[10:]').replace(''); + const kept = mdq(section.text()).query('blockquote[10:]').replace('').toString(); trimmedTitles.add(heading); return `${kept.trimEnd()}\n> ... and ${count - 10} more discoveries\n`; - }); + }) + .toString(); } return result.trim() || null; @@ -402,7 +404,7 @@ export class Planner extends PlannerBase implements Agent { if (this.scout && this.docsWeight > 0) { docsPromise = this.scout.collectDocs({ url: state.url, title: state.title, feature, excludeUrls: this.knowledgeTracker.applicationSpecUrls(state) }); } - let plannerResearch = mdq(research).query('code').replace(''); + let plannerResearch = mdq(research).query('code').replace('').toString(); plannerResearch = mdq(plannerResearch) .query('table') .replaceEach((table) => { @@ -413,7 +415,8 @@ export class Planner extends PlannerBase implements Agent { Type: r.Type || '', })); return jsonToTable(elementWithType, ['Element', 'Type']); - }); + }) + .toString(); const hasFocusedOverlay = hasFocusedSection(plannerResearch); const focusNote = hasFocusedOverlay ? "IMPORTANT: One section is marked as **Focused** — this is the user's current focus area. Concentrate testing on the Focused section FIRST — test all interactions inside it before planning tests for the rest of the page." : ''; diff --git a/src/ai/researcher.ts b/src/ai/researcher.ts index 358c48ed..09e08c61 100644 --- a/src/ai/researcher.ts +++ b/src/ai/researcher.ts @@ -313,7 +313,7 @@ export class Researcher extends ResearcherBase implements Agent { researchFile = saveResearch(researchState, result.text, combinedHtml); } - const summaryText = mdq(result.text).query('section2(/^summary/)').query('paragraph[0]').text().trim(); + const summaryText = mdq(result.text).query('section2(/^summary/i)').query('paragraph[0]').text().trim(); const summaryLine = summaryText.split('\n')[0]?.trim().slice(0, 200); if (summaryLine) this.experienceTracker.updateSummary(this.actionResult!, summaryLine); diff --git a/src/ai/researcher/deep-analysis.ts b/src/ai/researcher/deep-analysis.ts index a5414167..16d10a6d 100644 --- a/src/ai/researcher/deep-analysis.ts +++ b/src/ai/researcher/deep-analysis.ts @@ -126,7 +126,7 @@ export function WithDeepAnalysis(Base: T) { let updated: string; if (extQuery.count() > 0) { const existing = extQuery.text().trimEnd(); - updated = extQuery.replace(`${existing}\n\n${sectionMarkdown}\n`); + updated = extQuery.replace(`${existing}\n\n${sectionMarkdown}\n`).toString(); } else { updated = `${cached.trimEnd()}\n\n# Extended Research\n\n${sectionMarkdown}\n`; } @@ -539,7 +539,7 @@ export function WithDeepAnalysis(Base: T) { if (heading.count() === 0) heading = mdq(sectionMarkdown).query('h2[0]'); if (heading.count() === 0) return sectionMarkdown; - return heading.replace(`${heading.text().trimEnd()}\n\n> Container: '${containerCss}'\n\n`); + return heading.replace(`${heading.text().trimEnd()}\n\n> Container: '${containerCss}'\n\n`).toString(); } private _deduplicateExpandedSections(sections: string[]): string[] { diff --git a/src/ai/researcher/locators.ts b/src/ai/researcher/locators.ts index cfe0d278..e4746258 100644 --- a/src/ai/researcher/locators.ts +++ b/src/ai/researcher/locators.ts @@ -304,9 +304,9 @@ export function WithLocators(Base: T) { if (sectionQuery.count() === 0) sectionQuery = mdq(result.text).query(`section3(~"${escaped}")`); if (newCss) { - result.text = sectionQuery.query('blockquote[0]').setKeyValue('Container', `'${newCss}'`); + result.text = sectionQuery.query('blockquote[0]').setKeyValue('Container', `'${newCss}'`).toString(); } else { - result.text = sectionQuery.query('blockquote[0]').replace(''); + result.text = sectionQuery.query('blockquote[0]').replace('').toString(); result.text = result.text.replace(`${FOCUSED_MARKER}\n`, ''); } diff --git a/src/ai/researcher/pagination.ts b/src/ai/researcher/pagination.ts index af24a204..8e405ac0 100644 --- a/src/ai/researcher/pagination.ts +++ b/src/ai/researcher/pagination.ts @@ -58,7 +58,7 @@ export function WithPagination(Base: T) { let sectionQuery = mdq(result.text).query(`section2(~"${escaped}")`); if (sectionQuery.count() === 0) sectionQuery = mdq(result.text).query(`section3(~"${escaped}")`); if (sectionQuery.count() === 0) return; - result.text = sectionQuery.query('blockquote[0]').setKeyValue('Pagination', strategy); + result.text = sectionQuery.query('blockquote[0]').setKeyValue('Pagination', strategy).toString(); } }; } diff --git a/src/ai/researcher/research-result.ts b/src/ai/researcher/research-result.ts index 7eeece7b..a2cbbea0 100644 --- a/src/ai/researcher/research-result.ts +++ b/src/ai/researcher/research-result.ts @@ -52,9 +52,9 @@ export class ResearchResult { const escaped = section.name.replace(/"/g, '\\"'); let sectionQuery = mdq(this.text).query(`section2(~"${escaped}")`); if (sectionQuery.count() === 0) sectionQuery = mdq(this.text).query(`section3(~"${escaped}")`); - const updated = sectionQuery.query('table').replace(`${newTable.trimEnd()}\n`); + const updated = sectionQuery.query('table').replace(`${newTable.trimEnd()}\n`).toString(); if (updated === this.text) return; - section.rawMarkdown = mdq(section.rawMarkdown).query('table').replace(`${newTable.trimEnd()}\n`); + section.rawMarkdown = mdq(section.rawMarkdown).query('table').replace(`${newTable.trimEnd()}\n`).toString(); this.text = updated; } diff --git a/src/experience-tracker.ts b/src/experience-tracker.ts index 12690f0b..89b0368d 100644 --- a/src/experience-tracker.ts +++ b/src/experience-tracker.ts @@ -262,7 +262,7 @@ export class ExperienceTracker { while (content.split('\n').length > maxLines) { const sections = mdq(content).query('section2').each(); if (sections.length <= 1) break; - content = sections[sections.length - 1].replace(''); + content = sections[sections.length - 1].replace('').toString(); } return { ...experience, content }; }); @@ -286,7 +286,7 @@ export class ExperienceTracker { combined = renderAsHowTo(combined); if (options?.stripCode) { - combined = mdq(combined).query('code').replace(''); + combined = mdq(combined).query('code').replace('').toString(); } if (combined.trim()) results.push(combined.trim()); @@ -515,7 +515,8 @@ function renderAsHowTo(content: string): string { if (text.startsWith('FLOW:')) return `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`; if (text.startsWith('ACTION:')) return `## HOW to ${text.slice(7).trim()} (single-step)\n\n`; return heading.text(); - }); + }) + .toString(); } export interface ExperienceFile { diff --git a/src/utils/mdq/edit.ts b/src/utils/mdq/edit.ts index 1f819f38..95339f40 100644 --- a/src/utils/mdq/edit.ts +++ b/src/utils/mdq/edit.ts @@ -1,3 +1,5 @@ +import type { MatchedRange } from './query.ts'; + export function splitFrontmatter(source: string): FrontmatterSplit { if (!source.startsWith('---')) return { raw: '', body: source, offset: 0 }; const match = source.match(/^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/); @@ -5,6 +7,18 @@ export function splitFrontmatter(source: string): FrontmatterSplit { return { raw: match[1], body: source.slice(match[0].length), offset: match[0].length }; } +export function dedupeRanges(ranges: MatchedRange[]): MatchedRange[] { + const sorted = [...ranges].sort((a, b) => a.start - b.start); + const kept: MatchedRange[] = []; + let lastEnd = -1; + for (const range of sorted) { + if (range.start < lastEnd) continue; + kept.push(range); + lastEnd = range.start + range.length; + } + return kept; +} + export interface FrontmatterSplit { raw: string; body: string; diff --git a/src/utils/mdq/query.ts b/src/utils/mdq/query.ts index a1f9e404..a50c6c94 100644 --- a/src/utils/mdq/query.ts +++ b/src/utils/mdq/query.ts @@ -1,5 +1,5 @@ import { type Token, type Tokens, marked } from 'marked'; -import { splitFrontmatter } from './edit.ts'; +import { dedupeRanges, splitFrontmatter } from './edit.ts'; export { splitFrontmatter }; @@ -369,31 +369,56 @@ function executeSegments(candidates: MatchedRange[], segments: QuerySegment[]): return executeSegments(applyIndexSlice(matches, segment), remaining); } -export class MarkdownQuery { - private source: string; +export class MarkdownDoc { + protected source: string; + + constructor(source: string) { + this.source = source; + } + + query(selector: string): Selection { + const segments = parseQuery(selector); + const candidates = expandSectionRanges(buildTokenIndex(this.source)); + return new Selection(this.source, executeSegments(candidates, segments)); + } + + toString(): string { + return this.source; + } + + valueOf(): string { + return this.source; + } +} + +export class Selection extends MarkdownDoc { private matches: MatchedRange[]; constructor(source: string, matches?: MatchedRange[]) { - this.source = source; + super(source); this.matches = matches || buildTokenIndex(source); } - query(selector: string): MarkdownQuery { + query(selector: string): Selection { const segments = parseQuery(selector); const candidates = expandSectionRanges(this.matches); - const results = executeSegments(candidates, segments); - return new MarkdownQuery(this.source, results); + return new Selection(this.source, executeSegments(candidates, segments)); } text(): string { return this.matches.map((r) => this.source.slice(r.start, r.start + r.length)).join(''); } + toString(): string { + return this.text(); + } + + /** @deprecated Use text(). */ get(): string { return this.text(); } - toJson(): Record[] { + rows(): Record[] { const results: Record[] = []; for (const range of this.matches) { @@ -413,7 +438,12 @@ export class MarkdownQuery { return results; } - keyValue(): Record { + /** @deprecated Use rows(). */ + toJson(): Record[] { + return this.rows(); + } + + entries(): Record { const entries: Record = {}; for (const range of this.matches) { @@ -428,7 +458,12 @@ export class MarkdownQuery { return entries; } - setKeyValue(key: string, value: string | null): string { + /** @deprecated Use entries(). */ + keyValue(): Record { + return this.entries(); + } + + setEntry(key: string, value: string | null): MarkdownDoc { return this.replaceEach((match) => { const token = match.matches[0].token; const lines = getTokenText(token) @@ -446,76 +481,95 @@ export class MarkdownQuery { }); } - replace(content: string): string { - return this.replaceEach(() => content); + /** @deprecated Use setEntry(). */ + setKeyValue(key: string, value: string | null): MarkdownDoc { + return this.setEntry(key, value); } - replaceEach(replacer: (match: MarkdownQuery, index: number) => string): string { - const sorted = [...this.matches].sort((a, b) => a.start - b.start); - - const kept: MatchedRange[] = []; - let lastEnd = -1; - for (const range of sorted) { - if (range.start < lastEnd) continue; - kept.push(range); - lastEnd = range.start + range.length; - } + replace(content: Markdown): MarkdownDoc { + return this.replaceEach(() => content); + } - const replacements = kept.map((range, index) => replacer(new MarkdownQuery(this.source, [range]), index)); + replaceEach(replacer: (match: Selection, index: number) => Markdown): MarkdownDoc { + const kept = dedupeRanges(this.matches); + const replacements = kept.map((range, index) => String(replacer(new Selection(this.source, [range]), index))); let result = this.source; for (let i = kept.length - 1; i >= 0; i--) { const range = kept[i]; result = result.slice(0, range.start) + replacements[i] + result.slice(range.start + range.length); } - return result; + return new MarkdownDoc(result); } count(): number { return this.matches.length; } - first(): MarkdownQuery { - return new MarkdownQuery(this.source, this.matches.slice(0, 1)); + exists(): boolean { + return this.matches.length > 0; + } + + first(): Selection { + return new Selection(this.source, this.matches.slice(0, 1)); } - last(): MarkdownQuery { - return new MarkdownQuery(this.source, this.matches.slice(-1)); + last(): Selection { + return new Selection(this.source, this.matches.slice(-1)); } - before(): MarkdownQuery { - if (this.matches.length === 0) return new MarkdownQuery(this.source, []); + preceding(): Selection { + if (this.matches.length === 0) return new Selection(this.source, []); const cutoff = this.matches[0].start; - const allTokens = buildTokenIndex(this.source); - const beforeTokens = allTokens.filter((r) => r.start + r.length <= cutoff); - return new MarkdownQuery(this.source, beforeTokens); + return new Selection( + this.source, + buildTokenIndex(this.source).filter((r) => r.start + r.length <= cutoff) + ); } - after(): MarkdownQuery { - if (this.matches.length === 0) return new MarkdownQuery(this.source, []); + /** @deprecated Use preceding(). */ + before(): Selection { + return this.preceding(); + } + + following(): Selection { + if (this.matches.length === 0) return new Selection(this.source, []); const lastMatch = this.matches[this.matches.length - 1]; const cutoff = lastMatch.start + lastMatch.length; - const allTokens = buildTokenIndex(this.source); - const afterTokens = allTokens.filter((r) => r.start >= cutoff); - return new MarkdownQuery(this.source, afterTokens); + return new Selection( + this.source, + buildTokenIndex(this.source).filter((r) => r.start >= cutoff) + ); } - each(): MarkdownQuery[] { - return this.matches.map((m) => new MarkdownQuery(this.source, [m])); + /** @deprecated Use following(). */ + after(): Selection { + return this.following(); } - meta(): Array<{ type: string; depth: number | null; text: string }> { + each(): Selection[] { + return this.matches.map((m) => new Selection(this.source, [m])); + } + + nodes(): NodeInfo[] { return this.matches.map((range) => { const token = range.token as any; - let depth: number | null = null; - if (token.type === 'heading') depth = token.depth; - return { type: token.type, depth, text: getTokenText(range.token) }; + if (token.type !== 'heading') return { type: token.type, depth: null, text: getTokenText(range.token) }; + return { type: token.type, depth: token.depth, text: getTokenText(range.token) }; }); } + + /** @deprecated Use nodes(). */ + meta(): NodeInfo[] { + return this.nodes(); + } } -export function mdq(source: string): MarkdownQuery { - return new MarkdownQuery(source); +/** @deprecated Use Selection. */ +export const MarkdownQuery = Selection; + +export function mdq(source: Markdown): Selection { + return new Selection(String(source)); } export type SelectorType = 'section' | 'section1' | 'section2' | 'section3' | 'section4' | 'section5' | 'section6' | 'table' | 'heading' | 'paragraph' | 'list' | 'item' | 'code' | 'blockquote' | 'hr' | 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6'; @@ -540,3 +594,11 @@ export interface MatchedRange { trailing?: { start: number; length: number }; innerTokens?: MatchedRange[]; } + +export type Markdown = string | MarkdownDoc; + +export interface NodeInfo { + type: string; + depth: number | null; + text: string; +} diff --git a/tests/unit/mdq/query.test.ts b/tests/unit/mdq/query.test.ts index f44d2595..88ee1ffd 100644 --- a/tests/unit/mdq/query.test.ts +++ b/tests/unit/mdq/query.test.ts @@ -639,40 +639,42 @@ Not a section. describe('replace', () => { it('should replace matched content', () => { - const result = mdq(sampleMarkdown).query('heading("FAQ")').replace('## Questions\n'); + const result = String(mdq(sampleMarkdown).query('heading("FAQ")').replace('## Questions\n')); expect(result).toContain('## Questions'); expect(result).not.toContain('## FAQ'); }); it('should replace table', () => { - const result = mdq(sampleMarkdown).query('section("Rate Limiting") table').replace('No limits!\n'); + const result = String(mdq(sampleMarkdown).query('section("Rate Limiting") table').replace('No limits!\n')); expect(result).toContain('No limits!'); expect(result).not.toContain('1 hour'); expect(result).toContain('/users'); }); it('should replace section', () => { - const result = mdq(sampleMarkdown).query('section("FAQ")').replace('## FAQ\n\nNo questions.\n'); + const result = String(mdq(sampleMarkdown).query('section("FAQ")').replace('## FAQ\n\nNo questions.\n')); expect(result).toContain('No questions'); expect(result).not.toContain('blockquote'); }); it('should return source unchanged when no matches', () => { - const result = mdq(sampleMarkdown).query('heading("Nonexistent")').replace('replaced'); + const result = String(mdq(sampleMarkdown).query('heading("Nonexistent")').replace('replaced')); expect(result).toBe(sampleMarkdown); }); it('should handle overlapping ranges (keep outermost)', () => { const md = '## Parent\n\n### Child\n\nContent\n'; - const result = mdq(md).query('section').replace('REPLACED\n'); + const result = String(mdq(md).query('section').replace('REPLACED\n')); expect(result).toBe('REPLACED\n'); }); it('should replace each match without stale offsets', () => { const md = '## Short\n\nText\n\n## Much Longer Heading\n\nMore\n'; - const result = mdq(md) - .query('h2') - .replaceEach((heading, index) => `## ${index + 1}: ${heading.meta()[0].text}\n\n`); + const result = String( + mdq(md) + .query('h2') + .replaceEach((heading, index) => `## ${index + 1}: ${heading.meta()[0].text}\n\n`) + ); expect(result).toBe('## 1: Short\n\nText\n\n## 2: Much Longer Heading\n\nMore\n'); }); @@ -714,7 +716,7 @@ Not a section. }); it('should handle empty replace', () => { - const result = mdq('').query('heading').replace('x'); + const result = String(mdq('').query('heading').replace('x')); expect(result).toBe(''); }); @@ -769,7 +771,7 @@ Not a section. const block = "## S\n\n> Container: '.old'\n> Pagination: controls\n\ntext\n"; it('replaces an entry in place and keeps the others', () => { - const updated = mdq(block).query('blockquote[0]').setKeyValue('Container', "'.new'"); + const updated = String(mdq(block).query('blockquote[0]').setKeyValue('Container', "'.new'")); expect(updated).toBe("## S\n\n> Container: '.new'\n> Pagination: controls\n\ntext\n"); }); @@ -794,8 +796,34 @@ Not a section. }); it('leaves a plain paragraph unprefixed', () => { - const updated = mdq('Container: .old\n').query('paragraph[0]').setKeyValue('Container', '.new'); + const updated = String(mdq('Container: .old\n').query('paragraph[0]').setKeyValue('Container', '.new')); expect(updated).toBe('Container: .new'); }); }); }); + +describe('MarkdownDoc chaining', () => { + const md = '# T\n\n## A\n\npara\n\n## B\n\nother\n'; + + it('returns a MarkdownDoc from a write so edits chain', () => { + expect(mdq(md).query('h2("A")').replace('## Z\n\n').query('h2').count()).toBe(2); + }); + + it('stringifies to the full document', () => { + expect(mdq(md).query('h2("A")').replace('## Z\n\n').toString()).toContain('## Z'); + }); + + it('accepts a MarkdownDoc as a source', () => { + const doc = mdq(md).query('h2("A")').replace('## Z\n\n'); + expect(mdq(doc).query('h2').count()).toBe(2); + }); + + it('accepts a MarkdownDoc returned from a replaceEach callback', () => { + const out = mdq(md) + .query('h2') + .replaceEach((section) => mdq(section.text()).query('h2').replace('### x\n\n')) + .toString(); + expect(out).toContain('### x'); + expect(out).not.toContain('## A'); + }); +}); diff --git a/tests/unit/research-parser-pagination.test.ts b/tests/unit/research-parser-pagination.test.ts index 2c7e9f64..b07420ed 100644 --- a/tests/unit/research-parser-pagination.test.ts +++ b/tests/unit/research-parser-pagination.test.ts @@ -62,7 +62,7 @@ describe('pagination line', () => { }); describe('rewriting an entry', () => { - const rewrite = (markdown: string, key: string, value: string) => mdq(markdown).query('section2(~"Menu")').query('blockquote[0]').setKeyValue(key, value); + const rewrite = (markdown: string, key: string, value: string) => mdq(markdown).query('section2(~"Menu")').query('blockquote[0]').setKeyValue(key, value).toString(); it('leaves the blockquote readable', () => { const markdown = `## Menu\n\n> Container: '.old'\n\n| Element | ARIA | CSS | eidx |\n`; From 8488e6ed359591a435635c2e0bad0e1e49bc2cf7 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:23:07 +0300 Subject: [PATCH 08/21] feat(mdq): comment and html selectors, honest regex flags, loud selector errors comment matches block comments on their inner body, so anchored patterns like /^test/ work; html matches every html block on raw text. Regex flags are now honored instead of parsed and discarded, unknown selectors throw MdqSelectorError with a position instead of silently matching nothing, and table text-matching reaches cells rather than headers only. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- src/utils/mdq/query.ts | 65 +++++++++++++++++++--- tests/unit/mdq/query.test.ts | 8 +-- tests/unit/mdq/selectors.test.ts | 93 ++++++++++++++++++++++++++++++++ 3 files changed, 154 insertions(+), 12 deletions(-) create mode 100644 tests/unit/mdq/selectors.test.ts diff --git a/src/utils/mdq/query.ts b/src/utils/mdq/query.ts index a50c6c94..5f935f52 100644 --- a/src/utils/mdq/query.ts +++ b/src/utils/mdq/query.ts @@ -3,6 +3,43 @@ import { dedupeRanges, splitFrontmatter } from './edit.ts'; export { splitFrontmatter }; +export class MdqError extends Error {} + +export class MdqSelectorError extends MdqError { + index: number; + + constructor(message: string, index: number) { + super(message); + this.name = 'MdqSelectorError'; + this.index = index; + } +} + +export class MdqOperationError extends MdqError { + constructor(message: string) { + super(message); + this.name = 'MdqOperationError'; + } +} + +const SELECTORS = new Set(['section', 'heading', 'paragraph', 'table', 'list', 'item', 'code', 'blockquote', 'hr', 'html', 'comment']); + +function isKnownSelector(selector: string): boolean { + if (/^h[1-6]$/.test(selector)) return true; + if (/^section[1-6]?$/.test(selector)) return true; + return SELECTORS.has(selector); +} + +function isCommentToken(token: Token): boolean { + if (token.type !== 'html') return false; + return (((token as any).raw as string) || '').trimStart().startsWith('$/, '').trim(); +} + export function parseQuery(input: string): QuerySegment[] { const segments: QuerySegment[] = []; let pos = 0; @@ -69,8 +106,9 @@ export function parseQuery(input: string): QuerySegment[] { pos++; } if (pos < input.length) pos++; + const flagStart = pos; while (pos < input.length && /[gimsuy]/.test(input[pos])) pos++; - return { mode: 'regex', value, negated }; + return { mode: 'regex', value, negated, flags: input.slice(flagStart, pos) }; } const value = readQuotedString(); @@ -81,11 +119,11 @@ export function parseQuery(input: string): QuerySegment[] { skipWhitespace(); if (pos >= input.length) break; + if (peek() === '.') advance(); + const selectorStart = pos; const selector = readIdentifier(); - if (!selector) { - pos++; - continue; - } + if (!selector) throw new MdqSelectorError(`Unexpected character "${input[pos]}" in selector`, pos); + if (!isKnownSelector(selector)) throw new MdqSelectorError(`Unknown selector "${selector}"`, selectorStart); const segment: QuerySegment = { selector: selector as SelectorType, @@ -140,7 +178,7 @@ function matchText(text: string, matcher: TextMatcher): boolean { result = text.includes(matcher.value); break; case 'regex': - result = new RegExp(matcher.value, 'i').test(text); + result = new RegExp(matcher.value, matcher.flags || '').test(text); break; default: result = false; @@ -165,8 +203,11 @@ function getTokenText(token: Token): string { case 'blockquote': case 'list_item': return t.text || ''; + case 'html': + if (isCommentToken(token)) return commentBody(token); + return t.raw || ''; case 'table': - return (t.header || []).map((h: any) => h.text).join(', '); + return [...(t.header || []).map((h: any) => h.text), ...(t.rows || []).flatMap((row: any) => row.map((cell: any) => cell.text))].join(', '); default: return ''; } @@ -198,6 +239,7 @@ function selectorToTokenType(selector: string): string | null { list: 'list', blockquote: 'blockquote', hr: 'hr', + html: 'html', item: 'list_item', }; return map[selector] || null; @@ -344,6 +386,12 @@ function executeSegments(candidates: MatchedRange[], segments: QuerySegment[]): return results; } + if (segment.selector === 'comment') { + let comments = candidates.filter((r) => isCommentToken(r.token)); + if (segment.textMatch) comments = comments.filter((r) => matchText(commentBody(r.token), segment.textMatch!)); + return executeSegments(applyIndexSlice(comments, segment), remaining); + } + if (segment.selector === 'item') { let items = extractListItems(candidates); if (segment.textMatch) { @@ -572,12 +620,13 @@ export function mdq(source: Markdown): Selection { return new Selection(String(source)); } -export type SelectorType = 'section' | 'section1' | 'section2' | 'section3' | 'section4' | 'section5' | 'section6' | 'table' | 'heading' | 'paragraph' | 'list' | 'item' | 'code' | 'blockquote' | 'hr' | 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6'; +export type SelectorType = 'comment' | 'html' | 'section' | 'section1' | 'section2' | 'section3' | 'section4' | 'section5' | 'section6' | 'table' | 'heading' | 'paragraph' | 'list' | 'item' | 'code' | 'blockquote' | 'hr' | 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6'; export interface TextMatcher { mode: 'exact' | 'contains' | 'regex'; value: string; negated: boolean; + flags?: string; } export interface QuerySegment { diff --git a/tests/unit/mdq/query.test.ts b/tests/unit/mdq/query.test.ts index 88ee1ffd..64b3f8be 100644 --- a/tests/unit/mdq/query.test.ts +++ b/tests/unit/mdq/query.test.ts @@ -90,12 +90,12 @@ describe('Markdown Query (mdq)', () => { it('should parse regex text matcher', () => { const segments = parseQuery('heading(/api/)'); - expect(segments[0].textMatch).toEqual({ mode: 'regex', value: 'api', negated: false }); + expect(segments[0].textMatch).toEqual({ mode: 'regex', value: 'api', negated: false, flags: '' }); }); - it('should skip regex flags', () => { + it('should keep regex flags', () => { const segments = parseQuery('section(/^api$/i)'); - expect(segments[0].textMatch).toEqual({ mode: 'regex', value: '^api$', negated: false }); + expect(segments[0].textMatch).toEqual({ mode: 'regex', value: '^api$', negated: false, flags: 'i' }); }); it('should parse negated text matcher', () => { @@ -110,7 +110,7 @@ describe('Markdown Query (mdq)', () => { it('should parse negated regex matcher', () => { const segments = parseQuery('heading(!/api/)'); - expect(segments[0].textMatch).toEqual({ mode: 'regex', value: 'api', negated: true }); + expect(segments[0].textMatch).toEqual({ mode: 'regex', value: 'api', negated: true, flags: '' }); }); it('should parse numeric index', () => { diff --git a/tests/unit/mdq/selectors.test.ts b/tests/unit/mdq/selectors.test.ts new file mode 100644 index 00000000..7bf7e3a1 --- /dev/null +++ b/tests/unit/mdq/selectors.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'vitest'; +import { MdqSelectorError, mdq } from '../../../src/utils/mdq/query.ts'; + +const doc = ` + +## Plan + + + +para with comment + +
a block
+ +| Method | Path | +|--------|------| +| GET | /users | +`; + +describe('comment selector', () => { + it('matches block comments and not other html', () => { + expect(mdq(doc).query('comment').count()).toBe(2); + }); + + it('matches on the inner body so anchored patterns work', () => { + expect(mdq(doc).query('comment(/^test/)').count()).toBe(1); + }); + + it('exposes the inner body as node text, without the markers', () => { + expect(mdq(doc).query('comment[0]').nodes()[0].text).toBe('suite'); + }); + + it('keeps newlines inside a multi-line comment', () => { + expect(mdq(doc).query('comment(/^test/)').nodes()[0].text).toContain('\n'); + }); + + it('does not reach comments inline in a paragraph', () => { + expect(mdq(doc).query('comment(~"inline")').count()).toBe(0); + }); + + it('matches an exact single-line comment body', () => { + expect(mdq(doc).query('comment("suite")').count()).toBe(1); + }); +}); + +describe('html selector', () => { + it('matches every html block including comments', () => { + expect(mdq(doc).query('html').count()).toBe(3); + }); + + it('matches on raw text', () => { + expect(mdq(doc).query('html(~" { + it('honors an explicit i flag', () => { + expect(mdq('## Summary\n').query('h2(/^summary/i)').count()).toBe(1); + }); + + it('is case sensitive without the i flag', () => { + expect(mdq('## Summary\n').query('h2(/^summary/)').count()).toBe(0); + }); +}); + +describe('table text matching', () => { + it('matches cell content, not only headers', () => { + expect(mdq(doc).query('table(~"/users")').count()).toBe(1); + }); + + it('still matches header content', () => { + expect(mdq(doc).query('table(~"Method")').count()).toBe(1); + }); +}); + +describe('selector errors', () => { + it('throws on an unknown selector rather than matching nothing', () => { + expect(() => mdq(doc).query('secton("A")')).toThrow(MdqSelectorError); + }); + + it('reports where the problem is', () => { + try { + mdq(doc).query('h2("A") secton("B")'); + expect.unreachable(); + } catch (error: any) { + expect(error.index).toBe(8); + } + }); + + it('accepts a leading dot for jq muscle memory', () => { + expect(mdq('## A\n').query('.h2').count()).toBe(1); + }); +}); From 3aa1de89254906a6e52363013e2d128bfd746021 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:25:12 +0300 Subject: [PATCH 09/21] feat(mdq): value matchers, sugar layer, at/slice Matchers can be a string (exact), RegExp (own flags) or predicate, so a dynamic value no longer has to be escaped into a selector string. The sugar layer lives on MarkdownDoc and Selection inherits it, rather than a separate abstract base: Selection already extends MarkdownDoc and overrides query(), so the eleven one-liners dispatch correctly with no extra class. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- src/utils/mdq/query.ts | 89 +++++++++++++++++++++++-- tests/unit/mdq/sugar.test.ts | 122 +++++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+), 5 deletions(-) create mode 100644 tests/unit/mdq/sugar.test.ts diff --git a/src/utils/mdq/query.ts b/src/utils/mdq/query.ts index 5f935f52..30c13a32 100644 --- a/src/utils/mdq/query.ts +++ b/src/utils/mdq/query.ts @@ -167,6 +167,19 @@ function parseBound(value: string): number | undefined { return Number.parseInt(value, 10); } +function toTextMatcher(matcher: Matcher): TextMatcher { + if (typeof matcher === 'function') return { mode: 'predicate', value: '', negated: false, predicate: matcher }; + if (matcher instanceof RegExp) return { mode: 'regex', value: matcher.source, negated: false, flags: matcher.flags }; + return { mode: 'exact', value: matcher, negated: false }; +} + +function applyMatcher(segments: QuerySegment[], matcher?: Matcher): QuerySegment[] { + if (matcher === undefined) return segments; + if (segments.length === 0) return segments; + segments[segments.length - 1].textMatch = toTextMatcher(matcher); + return segments; +} + function matchText(text: string, matcher: TextMatcher): boolean { let result: boolean; @@ -180,6 +193,9 @@ function matchText(text: string, matcher: TextMatcher): boolean { case 'regex': result = new RegExp(matcher.value, matcher.flags || '').test(text); break; + case 'predicate': + result = matcher.predicate!(text); + break; default: result = false; } @@ -424,12 +440,57 @@ export class MarkdownDoc { this.source = source; } - query(selector: string): Selection { - const segments = parseQuery(selector); + query(selector: string, matcher?: Matcher): Selection { + const segments = applyMatcher(parseQuery(selector), matcher); const candidates = expandSectionRanges(buildTokenIndex(this.source)); return new Selection(this.source, executeSegments(candidates, segments)); } + section(matcher?: Matcher, options?: SelectorOptions): Selection { + return this.query(`section${options?.depth || ''}`, matcher); + } + + heading(matcher?: Matcher, options?: SelectorOptions): Selection { + if (options?.depth) return this.query(`h${options.depth}`, matcher); + return this.query('heading', matcher); + } + + paragraph(matcher?: Matcher): Selection { + return this.query('paragraph', matcher); + } + + table(matcher?: Matcher): Selection { + return this.query('table', matcher); + } + + list(matcher?: Matcher): Selection { + return this.query('list', matcher); + } + + item(matcher?: Matcher): Selection { + return this.query('item', matcher); + } + + code(matcher?: Matcher): Selection { + return this.query('code', matcher); + } + + blockquote(matcher?: Matcher): Selection { + return this.query('blockquote', matcher); + } + + comment(matcher?: Matcher): Selection { + return this.query('comment', matcher); + } + + html(matcher?: Matcher): Selection { + return this.query('html', matcher); + } + + hr(): Selection { + return this.query('hr'); + } + toString(): string { return this.source; } @@ -447,8 +508,8 @@ export class Selection extends MarkdownDoc { this.matches = matches || buildTokenIndex(source); } - query(selector: string): Selection { - const segments = parseQuery(selector); + query(selector: string, matcher?: Matcher): Selection { + const segments = applyMatcher(parseQuery(selector), matcher); const candidates = expandSectionRanges(this.matches); return new Selection(this.source, executeSegments(candidates, segments)); } @@ -558,6 +619,17 @@ export class Selection extends MarkdownDoc { return this.matches.length > 0; } + at(index: number): Selection { + let resolved = index; + if (resolved < 0) resolved = this.matches.length + resolved; + if (resolved < 0 || resolved >= this.matches.length) return new Selection(this.source, []); + return new Selection(this.source, [this.matches[resolved]]); + } + + slice(from?: number, to?: number): Selection { + return new Selection(this.source, this.matches.slice(from, to)); + } + first(): Selection { return new Selection(this.source, this.matches.slice(0, 1)); } @@ -623,10 +695,17 @@ export function mdq(source: Markdown): Selection { export type SelectorType = 'comment' | 'html' | 'section' | 'section1' | 'section2' | 'section3' | 'section4' | 'section5' | 'section6' | 'table' | 'heading' | 'paragraph' | 'list' | 'item' | 'code' | 'blockquote' | 'hr' | 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6'; export interface TextMatcher { - mode: 'exact' | 'contains' | 'regex'; + mode: 'exact' | 'contains' | 'regex' | 'predicate'; value: string; negated: boolean; flags?: string; + predicate?: (text: string) => boolean; +} + +export type Matcher = string | RegExp | ((text: string) => boolean); + +export interface SelectorOptions { + depth?: 1 | 2 | 3 | 4 | 5 | 6; } export interface QuerySegment { diff --git a/tests/unit/mdq/sugar.test.ts b/tests/unit/mdq/sugar.test.ts new file mode 100644 index 00000000..83787993 --- /dev/null +++ b/tests/unit/mdq/sugar.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it } from 'vitest'; +import { mdq } from '../../../src/utils/mdq/query.ts'; + +const doc = `## API "v2" + +| Method | Path | +|--------|------| +| GET | /users | + +## Settings + +- Option A +- Option B + + +`; + +describe('matchers as values', () => { + it('matches a string exactly', () => { + expect(mdq(doc).query('h2', 'Settings').count()).toBe(1); + expect(mdq(doc).query('h2', 'Setting').count()).toBe(0); + }); + + it('matches a RegExp honoring its flags', () => { + expect( + mdq(doc) + .query('h2', /^settings$/i) + .count() + ).toBe(1); + expect( + mdq(doc) + .query('h2', /^settings$/) + .count() + ).toBe(0); + }); + + it('matches a predicate', () => { + expect( + mdq(doc) + .query('h2', (t) => t.startsWith('API')) + .count() + ).toBe(1); + }); + + it('needs no escaping for a value containing quotes', () => { + expect(mdq(doc).query('h2', 'API "v2"').count()).toBe(1); + }); +}); + +describe('sugar', () => { + it('is equivalent to the query form', () => { + expect(mdq(doc).heading('Settings').text()).toBe(mdq(doc).query('heading', 'Settings').text()); + }); + + it('takes a depth option', () => { + expect(mdq(doc).section('Settings', { depth: 2 }).text()).toBe(mdq(doc).query('section2("Settings")').text()); + }); + + it('reads comments', () => { + expect(mdq(doc).comment(/^test/).count()).toBe(1); + }); + + it('chains from a Selection', () => { + expect(mdq(doc).section('API "v2"').table().rows()[0].Path).toBe('/users'); + }); + + it('takes no matcher', () => { + expect(mdq(doc).table().count()).toBe(1); + }); + + it('is available on a document returned by a write', () => { + expect(mdq(doc).table().replace('gone\n').heading().count()).toBe(2); + }); +}); + +describe('at and slice', () => { + it('selects by index like the DSL', () => { + expect(mdq(doc).heading().at(0).text()).toBe(mdq(doc).query('heading[0]').text()); + }); + + it('supports a negative index', () => { + expect(mdq(doc).heading().at(-1).text()).toContain('Settings'); + }); + + it('returns nothing for an out-of-bounds index', () => { + expect(mdq(doc).heading().at(99).count()).toBe(0); + expect(mdq(doc).heading().at(-99).count()).toBe(0); + }); + + it('slices like the DSL', () => { + expect(mdq(doc).item().slice(1).count()).toBe(1); + }); +}); + +describe('exists', () => { + it('is true when something matched', () => { + expect(mdq(doc).heading('Settings').exists()).toBe(true); + }); + + it('is false when nothing matched', () => { + expect(mdq(doc).heading('Nope').exists()).toBe(false); + }); +}); + +describe('canonical read names', () => { + it('rows matches the deprecated toJson', () => { + expect(mdq(doc).table().rows()).toEqual(mdq(doc).table().toJson()); + }); + + it('preceding matches the deprecated before', () => { + expect(mdq(doc).heading('Settings').preceding().text()).toBe(mdq(doc).query('heading("Settings")').before().text()); + }); + + it('following matches the deprecated after', () => { + expect(mdq(doc).heading('API "v2"').following().text()).toBe(mdq(doc).query('heading(~"API")').after().text()); + }); + + it('entries matches the deprecated keyValue', () => { + const block = mdq('> Container: .x\n').query('blockquote[0]'); + expect(block.entries()).toEqual(block.keyValue()); + }); +}); From 3aba5b7045211d1cd3cbdbccf1ee48bee6d9bb4e Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:28:04 +0300 Subject: [PATCH 10/21] feat(mdq): remove and insert verbs with whitespace normalization remove() takes a node plus its adjacent space token and collapses only at end-of-document, because marked bakes separators into heading and document-final paragraph raws but emits them as siblings elsewhere. insertAt normalizes only the seam, so blank lines inside fenced code blocks survive. mdq() now returns MarkdownDoc rather than Selection, so document-level append/prepend are reachable; blocks() selects every top-level block for the one caller that read the whole document as a selection. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- src/knowledge-tracker.ts | 2 +- src/utils/mdq/edit.ts | 49 +++++++++++++++++ src/utils/mdq/query.ts | 59 ++++++++++++++++++-- tests/unit/mdq/edit.test.ts | 104 ++++++++++++++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 4 deletions(-) create mode 100644 tests/unit/mdq/edit.test.ts diff --git a/src/knowledge-tracker.ts b/src/knowledge-tracker.ts index 24d05c9e..bbe6009f 100644 --- a/src/knowledge-tracker.ts +++ b/src/knowledge-tracker.ts @@ -214,7 +214,7 @@ export class KnowledgeTracker { return this.knowledgeFiles.map((knowledge) => { const content = knowledge.content.trim(); - const firstLine = mdq(content).meta()[0]?.text.split('\n')[0]?.trim() || ''; + const firstLine = mdq(content).blocks().nodes()[0]?.text.split('\n')[0]?.trim() || ''; return { url: knowledge.url || knowledge.endpoint || '', firstLine, diff --git a/src/utils/mdq/edit.ts b/src/utils/mdq/edit.ts index 95339f40..b213155a 100644 --- a/src/utils/mdq/edit.ts +++ b/src/utils/mdq/edit.ts @@ -19,6 +19,55 @@ export function dedupeRanges(ranges: MatchedRange[]): MatchedRange[] { return kept; } +export function blockEnd(range: MatchedRange): number { + if (range.trailing) return range.trailing.start + range.trailing.length; + return range.start + range.length; +} + +export function normalizeBlock(markdown: string): string { + return `${markdown.replace(/\s+$/, '')}\n`; +} + +export function removeRanges(source: string, ranges: MatchedRange[]): string { + const ordered = dedupeRanges(ranges); + let result = source; + for (let i = ordered.length - 1; i >= 0; i--) { + const range = ordered[i]; + const head = result.slice(0, range.start); + const tail = result.slice(blockEnd(range)); + if (tail) { + result = head + tail; + continue; + } + if (!head) { + result = ''; + continue; + } + result = `${head.replace(/\n+$/, '')}\n`; + } + return result; +} + +export function insertAt(source: string, offset: number, markdown: string): string { + const block = normalizeBlock(markdown); + const before = source.slice(0, offset).replace(/\n+$/, ''); + const after = source.slice(offset).replace(/^\n+/, ''); + if (!before) return `${block}\n${after}`; + if (!after) return `${before}\n\n${block}`; + return `${before}\n\n${block}\n${after}`; +} + +export function spliceRanges(source: string, ranges: MatchedRange[], render: (range: MatchedRange, index: number) => string): string { + const ordered = dedupeRanges(ranges); + const rendered = ordered.map(render); + let result = source; + for (let i = ordered.length - 1; i >= 0; i--) { + const range = ordered[i]; + result = result.slice(0, range.start) + rendered[i] + result.slice(range.start + range.length); + } + return result; +} + export interface FrontmatterSplit { raw: string; body: string; diff --git a/src/utils/mdq/query.ts b/src/utils/mdq/query.ts index 30c13a32..177b1b39 100644 --- a/src/utils/mdq/query.ts +++ b/src/utils/mdq/query.ts @@ -1,5 +1,5 @@ import { type Token, type Tokens, marked } from 'marked'; -import { dedupeRanges, splitFrontmatter } from './edit.ts'; +import { blockEnd, dedupeRanges, insertAt, removeRanges, splitFrontmatter } from './edit.ts'; export { splitFrontmatter }; @@ -446,6 +446,10 @@ export class MarkdownDoc { return new Selection(this.source, executeSegments(candidates, segments)); } + blocks(): Selection { + return new Selection(this.source); + } + section(matcher?: Matcher, options?: SelectorOptions): Selection { return this.query(`section${options?.depth || ''}`, matcher); } @@ -491,6 +495,14 @@ export class MarkdownDoc { return this.query('hr'); } + append(markdown: Markdown): MarkdownDoc { + return new MarkdownDoc(insertAt(this.source, this.source.length, String(markdown))); + } + + prepend(markdown: Markdown): MarkdownDoc { + return new MarkdownDoc(insertAt(this.source, splitFrontmatter(this.source).offset, String(markdown))); + } + toString(): string { return this.source; } @@ -595,6 +607,26 @@ export class Selection extends MarkdownDoc { return this.setEntry(key, value); } + remove(): MarkdownDoc { + return new MarkdownDoc(removeRanges(this.source, this.matches)); + } + + insertBefore(markdown: Markdown): MarkdownDoc { + return this.insertEach((range) => range.start, markdown); + } + + insertAfter(markdown: Markdown): MarkdownDoc { + return this.insertEach((range) => blockEnd(range), markdown); + } + + prepend(markdown: Markdown): MarkdownDoc { + return this.insertEach((range) => this.containerStart(range), markdown); + } + + append(markdown: Markdown): MarkdownDoc { + return this.insertEach((range) => this.containerEnd(range), markdown); + } + replace(content: Markdown): MarkdownDoc { return this.replaceEach(() => content); } @@ -683,13 +715,34 @@ export class Selection extends MarkdownDoc { meta(): NodeInfo[] { return this.nodes(); } + + private insertEach(offsetOf: (range: MatchedRange) => number, markdown: Markdown): MarkdownDoc { + const offsets = this.matches.map(offsetOf).sort((a, b) => a - b); + let result = this.source; + for (let i = offsets.length - 1; i >= 0; i--) { + result = insertAt(result, offsets[i], String(markdown)); + } + return new MarkdownDoc(result); + } + + private containerStart(range: MatchedRange): number { + if (!range.innerTokens) throw new MdqOperationError(`prepend needs a section or list, got ${range.token.type}`); + return range.start + (((range.token as any).raw as string) || '').length; + } + + private containerEnd(range: MatchedRange): number { + if (!range.innerTokens) throw new MdqOperationError(`append needs a section or list, got ${range.token.type}`); + const last = range.innerTokens[range.innerTokens.length - 1]; + if (!last) return this.containerStart(range); + return blockEnd(last); + } } /** @deprecated Use Selection. */ export const MarkdownQuery = Selection; -export function mdq(source: Markdown): Selection { - return new Selection(String(source)); +export function mdq(source: Markdown): MarkdownDoc { + return new MarkdownDoc(String(source)); } export type SelectorType = 'comment' | 'html' | 'section' | 'section1' | 'section2' | 'section3' | 'section4' | 'section5' | 'section6' | 'table' | 'heading' | 'paragraph' | 'list' | 'item' | 'code' | 'blockquote' | 'hr' | 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6'; diff --git a/tests/unit/mdq/edit.test.ts b/tests/unit/mdq/edit.test.ts new file mode 100644 index 00000000..7d953c06 --- /dev/null +++ b/tests/unit/mdq/edit.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from 'vitest'; +import { mdq } from '../../../src/utils/mdq/query.ts'; + +describe('remove', () => { + it('takes a paragraph and its separator, leaving no crater', () => { + expect(mdq('# A\n\nfirst\n\nsecond\n').query('paragraph("first")').remove().toString()).toBe('# A\n\nsecond\n'); + }); + + it('takes a heading with its baked-in separator', () => { + expect(mdq('# A\n\n## B\n\ntext\n').query('h2').remove().toString()).toBe('# A\n\ntext\n'); + }); + + it('takes the leading separator when the node is last', () => { + expect(mdq('# A\n\nlast\n').query('paragraph').remove().toString()).toBe('# A\n'); + }); + + it('removes a whole section including its children', () => { + expect(mdq('## A\n\nx\n\n## B\n\ny\n').query('section("A")').remove().toString()).toBe('## B\n\ny\n'); + }); + + it('removes every match', () => { + expect(mdq('# T\n\n```js\na\n```\n\ntext\n\n```js\nb\n```\n').query('code').remove().toString()).toBe('# T\n\ntext\n'); + }); + + it('returns the document unchanged when nothing matches', () => { + const src = '# A\n\ntext\n'; + expect(mdq(src).query('h5').remove().toString()).toBe(src); + }); + + it('never rewrites blank lines inside a fenced code block', () => { + const src = '# A\n\n```js\na\n\n\nb\n```\n\ngone\n'; + expect(mdq(src).query('paragraph("gone")').remove().toString()).toContain('a\n\n\nb'); + }); +}); + +describe('insertBefore and insertAfter', () => { + it('inserts a sibling before a node', () => { + expect(mdq('## B\n\ntext\n').query('h2').insertBefore('## A\n').toString()).toBe('## A\n\n## B\n\ntext\n'); + }); + + it('inserts a sibling after a node', () => { + expect(mdq('## A\n\ntext\n').query('h2').insertAfter('## B\n').toString()).toBe('## A\n\n## B\n\ntext\n'); + }); + + it('normalizes an insert that already ends with blank lines', () => { + expect(mdq('## A\n\ntext\n').query('h2').insertAfter('## B\n\n\n\n').toString()).toBe('## A\n\n## B\n\ntext\n'); + }); + + it('normalizes an insert with no trailing newline', () => { + expect(mdq('## A\n\ntext\n').query('h2').insertAfter('## B').toString()).toBe('## A\n\n## B\n\ntext\n'); + }); + + it('accepts a MarkdownDoc', () => { + expect(mdq('## A\n\ntext\n').query('h2').insertAfter(mdq('## B\n')).toString()).toContain('## B'); + }); + + it('never rewrites blank lines inside a fenced code block', () => { + const src = '```js\na\n\n\nb\n```\n\n## A\n'; + expect(mdq(src).query('h2').insertAfter('## B\n').toString()).toContain('a\n\n\nb'); + }); +}); + +describe('prepend and append on a section', () => { + const src = '## A\n\nfirst\n\n## B\n\nother\n'; + + it('appends inside the section, before the next same-depth heading', () => { + expect(mdq(src).query('section("A")').append('last\n').toString()).toBe('## A\n\nfirst\n\nlast\n\n## B\n\nother\n'); + }); + + it('prepends directly after the section heading', () => { + expect(mdq(src).query('section("A")').prepend('intro\n').toString()).toBe('## A\n\nintro\n\nfirst\n\n## B\n\nother\n'); + }); + + it('appends at the end of the document when the section is last', () => { + expect(mdq(src).query('section("B")').append('tail\n').toString()).toBe('## A\n\nfirst\n\n## B\n\nother\n\ntail\n'); + }); + + it('throws when applied to a leaf node', () => { + expect(() => mdq(src).query('paragraph[0]').append('x\n')).toThrow(); + }); +}); + +describe('document-level append and prepend', () => { + it('appends a block at the end', () => { + expect(mdq('# A\n\ntext\n').append('## New\n').toString()).toBe('# A\n\ntext\n\n## New\n'); + }); + + it('prepends a block at the start', () => { + expect(mdq('# A\n\ntext\n').prepend('> note\n').toString()).toBe('> note\n\n# A\n\ntext\n'); + }); + + it('prepends after frontmatter, never before it', () => { + const out = mdq('---\nurl: /x\n---\n\n# A\n').prepend('> note\n').toString(); + expect(out.startsWith('---\nurl: /x\n---\n')).toBe(true); + expect(out).toContain('> note'); + }); +}); + +describe('chained edits', () => { + it('composes several writes in one expression', () => { + const out = mdq('## A\n\nfirst\n\n## B\n\nother\n').query('section("A")').append('added\n').query('paragraph("other")').remove().toString(); + expect(out).toBe('## A\n\nfirst\n\nadded\n\n## B\n'); + }); +}); From 63478485eb1d24ca2d15dbc5dc34d09b63b47fb9 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:29:07 +0300 Subject: [PATCH 11/21] feat(mdq): addRow and addItem structural inserts addRow re-renders the table so column pipes stay aligned and preserves alignment markers; addItem copies the list's existing marker and indent and continues an ordered list's numbering. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- src/utils/mdq/edit.ts | 24 ++++++++++++ src/utils/mdq/query.ts | 24 +++++++++++- tests/unit/mdq/structural.test.ts | 65 +++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 tests/unit/mdq/structural.test.ts diff --git a/src/utils/mdq/edit.ts b/src/utils/mdq/edit.ts index b213155a..0d6c24cf 100644 --- a/src/utils/mdq/edit.ts +++ b/src/utils/mdq/edit.ts @@ -68,6 +68,30 @@ export function spliceRanges(source: string, ranges: MatchedRange[], render: (ra return result; } +export function renderTable(headers: string[], rows: string[][], align: (string | null)[]): string { + const widths = headers.map((header, index) => Math.max(header.length, 3, ...rows.map((row) => (row[index] || '').length))); + const line = (cells: string[]) => `| ${cells.map((cell, index) => (cell || '').padEnd(widths[index])).join(' | ')} |`; + const divider = `| ${widths.map((width, index) => dashes(align[index], width)).join(' | ')} |`; + return `${[line(headers), divider, ...rows.map(line)].join('\n')}\n`; +} + +export function renderItem(listRaw: string, text: string): string { + const lines = listRaw.split('\n').filter((line) => line.trim()); + const last = lines[lines.length - 1] || '- x'; + const ordered = last.match(/^(\s*)(\d+)([.)])\s/); + if (ordered) return `${ordered[1]}${Number.parseInt(ordered[2], 10) + 1}${ordered[3]} ${text}`; + const bullet = last.match(/^(\s*)([-*+])\s/); + if (!bullet) return `- ${text}`; + return `${bullet[1]}${bullet[2]} ${text}`; +} + +function dashes(alignment: string | null, width: number): string { + if (alignment === 'center') return `:${'-'.repeat(Math.max(width - 2, 1))}:`; + if (alignment === 'left') return `:${'-'.repeat(Math.max(width - 1, 1))}`; + if (alignment === 'right') return `${'-'.repeat(Math.max(width - 1, 1))}:`; + return '-'.repeat(width); +} + export interface FrontmatterSplit { raw: string; body: string; diff --git a/src/utils/mdq/query.ts b/src/utils/mdq/query.ts index 177b1b39..18908519 100644 --- a/src/utils/mdq/query.ts +++ b/src/utils/mdq/query.ts @@ -1,5 +1,5 @@ import { type Token, type Tokens, marked } from 'marked'; -import { blockEnd, dedupeRanges, insertAt, removeRanges, splitFrontmatter } from './edit.ts'; +import { blockEnd, dedupeRanges, insertAt, removeRanges, renderItem, renderTable, spliceRanges, splitFrontmatter } from './edit.ts'; export { splitFrontmatter }; @@ -627,6 +627,28 @@ export class Selection extends MarkdownDoc { return this.insertEach((range) => this.containerEnd(range), markdown); } + addRow(row: Record): MarkdownDoc { + return new MarkdownDoc( + spliceRanges(this.source, this.matches, (range) => { + if (range.token.type !== 'table') throw new MdqOperationError(`addRow needs a table, got ${range.token.type}`); + const table = range.token as Tokens.Table; + const headers = table.header.map((cell) => cell.text); + const existing = table.rows.map((cells) => headers.map((_, index) => cells[index]?.text || '')); + return renderTable(headers, [...existing, headers.map((header) => row[header] || '')], table.align); + }) + ); + } + + addItem(text: string): MarkdownDoc { + return new MarkdownDoc( + spliceRanges(this.source, this.matches, (range) => { + if (range.token.type !== 'list') throw new MdqOperationError(`addItem needs a list, got ${range.token.type}`); + const raw = (((range.token as any).raw as string) || '').replace(/\s+$/, ''); + return `${raw}\n${renderItem(raw, text)}\n`; + }) + ); + } + replace(content: Markdown): MarkdownDoc { return this.replaceEach(() => content); } diff --git a/tests/unit/mdq/structural.test.ts b/tests/unit/mdq/structural.test.ts new file mode 100644 index 00000000..6e4ff3b4 --- /dev/null +++ b/tests/unit/mdq/structural.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest'; +import { mdq } from '../../../src/utils/mdq/query.ts'; + +const table = `| Method | Path | +|--------|------| +| GET | /users | +`; + +describe('addRow', () => { + it('appends a row and re-aligns every column', () => { + expect(mdq(table).query('table').addRow({ Method: 'POST', Path: '/sessions' }).toString()).toBe(['| Method | Path |', '| ------ | --------- |', '| GET | /users |', '| POST | /sessions |', ''].join('\n')); + }); + + it('round-trips through rows()', () => { + const out = mdq(table).query('table').addRow({ Method: 'POST', Path: '/sessions' }); + expect(mdq(out).query('table').rows()).toEqual([ + { Method: 'GET', Path: '/users' }, + { Method: 'POST', Path: '/sessions' }, + ]); + }); + + it('leaves a column blank when the object omits it', () => { + const out = mdq(table).query('table').addRow({ Method: 'PUT' }); + expect(mdq(out).query('table').rows()[1]).toEqual({ Method: 'PUT', Path: '' }); + }); + + it('ignores keys that are not columns', () => { + const out = mdq(table).query('table').addRow({ Method: 'PUT', Nope: 'x' }); + expect(mdq(out).query('table').rows()[1].Method).toBe('PUT'); + expect(mdq(out).query('table').text()).not.toContain('Nope'); + }); + + it('preserves column alignment markers', () => { + const aligned = '| A | B |\n|:--|--:|\n| 1 | 2 |\n'; + const out = mdq(aligned).query('table').addRow({ A: '3', B: '4' }).toString(); + expect(out).toContain(':--'); + expect(out).toContain('--:'); + }); + + it('throws on a non-table node', () => { + expect(() => mdq('para\n').query('paragraph').addRow({ a: 'b' })).toThrow(); + }); +}); + +describe('addItem', () => { + it('copies a dash marker', () => { + expect(mdq('- a\n- b\n').query('list').addItem('c').toString()).toBe('- a\n- b\n- c\n'); + }); + + it('copies a star marker', () => { + expect(mdq('* a\n* b\n').query('list').addItem('c').toString()).toBe('* a\n* b\n* c\n'); + }); + + it('continues an ordered list', () => { + expect(mdq('1. a\n2. b\n').query('list').addItem('c').toString()).toBe('1. a\n2. b\n3. c\n'); + }); + + it('preserves indentation', () => { + expect(mdq(' - a\n - b\n').query('list').addItem('c').toString()).toBe(' - a\n - b\n - c\n'); + }); + + it('throws on a non-list node', () => { + expect(() => mdq('para\n').query('paragraph').addItem('x')).toThrow(); + }); +}); From 0036d7bc6b0c9d6c03a0357b7c8da20497dbc51e Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:30:33 +0300 Subject: [PATCH 12/21] feat(mdq): frontmatter read/write via yaml Document API, setEntry Uses YAML.parseDocument rather than parse/stringify, so nested maps, lists and block scalars parse correctly and comments survive a write. Verified against the knowledge and experience formats CLAUDE.md documents: raw marked lexes their --- block as a setext h2 titled 'url: /login...', mdq reports zero headings and typed frontmatter. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- src/utils/mdq/edit.ts | 17 ++++++++ src/utils/mdq/query.ts | 10 ++++- tests/unit/mdq/frontmatter.test.ts | 66 +++++++++++++++++++++++++++++- 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/src/utils/mdq/edit.ts b/src/utils/mdq/edit.ts index 0d6c24cf..ce66b352 100644 --- a/src/utils/mdq/edit.ts +++ b/src/utils/mdq/edit.ts @@ -1,3 +1,4 @@ +import YAML from 'yaml'; import type { MatchedRange } from './query.ts'; export function splitFrontmatter(source: string): FrontmatterSplit { @@ -92,6 +93,22 @@ function dashes(alignment: string | null, width: number): string { return '-'.repeat(width); } +export function readFrontmatter(source: string): Record { + const { raw } = splitFrontmatter(source); + if (!raw) return {}; + return (YAML.parseDocument(raw).toJS() as Record) || {}; +} + +export function writeFrontmatter(source: string, key: string, value: unknown): string { + const { raw, body, offset } = splitFrontmatter(source); + const document = raw ? YAML.parseDocument(raw) : new YAML.Document({}); + if (value === null) document.delete(key); + if (value !== null) document.set(key, value); + const rendered = document.toString().replace(/\s+$/, ''); + if (!offset) return `---\n${rendered}\n---\n\n${source}`; + return `---\n${rendered}\n---\n${body}`; +} + export interface FrontmatterSplit { raw: string; body: string; diff --git a/src/utils/mdq/query.ts b/src/utils/mdq/query.ts index 18908519..81fe9a00 100644 --- a/src/utils/mdq/query.ts +++ b/src/utils/mdq/query.ts @@ -1,5 +1,5 @@ import { type Token, type Tokens, marked } from 'marked'; -import { blockEnd, dedupeRanges, insertAt, removeRanges, renderItem, renderTable, spliceRanges, splitFrontmatter } from './edit.ts'; +import { blockEnd, dedupeRanges, insertAt, readFrontmatter, removeRanges, renderItem, renderTable, spliceRanges, splitFrontmatter, writeFrontmatter } from './edit.ts'; export { splitFrontmatter }; @@ -446,6 +446,14 @@ export class MarkdownDoc { return new Selection(this.source, executeSegments(candidates, segments)); } + frontmatter(): Record { + return readFrontmatter(this.source); + } + + setFrontmatter(key: string, value: unknown): MarkdownDoc { + return new MarkdownDoc(writeFrontmatter(this.source, key, value)); + } + blocks(): Selection { return new Selection(this.source); } diff --git a/tests/unit/mdq/frontmatter.test.ts b/tests/unit/mdq/frontmatter.test.ts index ccace300..1cc7c150 100644 --- a/tests/unit/mdq/frontmatter.test.ts +++ b/tests/unit/mdq/frontmatter.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import { splitFrontmatter } from '../../../src/utils/mdq/edit.ts'; -import { buildTokenIndex } from '../../../src/utils/mdq/query.ts'; +import { buildTokenIndex, mdq } from '../../../src/utils/mdq/query.ts'; describe('splitFrontmatter', () => { it('splits a leading yaml block from the body', () => { @@ -60,3 +60,67 @@ describe('buildTokenIndex', () => { expect(buildTokenIndex('a\n\nb\n\nc\n').some((r) => r.token.type === 'space')).toBe(false); }); }); + +describe('frontmatter API', () => { + const src = '---\n# a leading comment\nurl: /login\nwait: 1000\ntags:\n - auth\n - smoke\n---\n\n# Title\n'; + + it('reads typed scalars, lists and nested maps', () => { + expect(mdq(src).frontmatter()).toEqual({ url: '/login', wait: 1000, tags: ['auth', 'smoke'] }); + }); + + it('returns an empty object when there is no frontmatter', () => { + expect(mdq('# Title\n').frontmatter()).toEqual({}); + }); + + it('updates a key in place', () => { + expect(mdq(src).setFrontmatter('wait', 2000).frontmatter().wait).toBe(2000); + }); + + it('preserves comments through a write', () => { + expect(mdq(src).setFrontmatter('wait', 2000).toString()).toContain('# a leading comment'); + }); + + it('preserves the body exactly', () => { + expect(mdq(src).setFrontmatter('wait', 2000).toString()).toContain('# Title'); + }); + + it('adds a key that was not there', () => { + expect(mdq(src).setFrontmatter('region', 'sidebar').frontmatter().region).toBe('sidebar'); + }); + + it('deletes a key when the value is null', () => { + expect(mdq(src).setFrontmatter('wait', null).frontmatter().wait).toBeUndefined(); + }); + + it('creates a frontmatter block on a document that has none', () => { + const out = mdq('# Title\n').setFrontmatter('url', '/x'); + expect(out.frontmatter()).toEqual({ url: '/x' }); + expect(out.toString()).toContain('# Title'); + }); + + it('keeps body queries blind to frontmatter after a write', () => { + expect(mdq(src).setFrontmatter('wait', 2000).query('h2').count()).toBe(0); + }); +}); + +describe('entries and setEntry', () => { + const block = "## S\n\n> Container: '.old'\n> Pagination: controls\n\ntext\n"; + + it('reads every entry of a blockquote without its markers', () => { + expect(mdq(block).query('blockquote[0]').entries()).toEqual({ container: "'.old'", pagination: 'controls' }); + }); + + it('replaces an entry in place and keeps the others', () => { + expect(mdq(block).query('blockquote[0]').setEntry('Container', "'.new'").toString()).toBe("## S\n\n> Container: '.new'\n> Pagination: controls\n\ntext\n"); + }); + + it('appends an entry that was not there', () => { + const out = mdq(block).query('blockquote[0]').setEntry('Region', 'sidebar'); + expect(mdq(out).query('blockquote[0]').entries().region).toBe('sidebar'); + }); + + it('removes an entry when the value is null', () => { + const out = mdq(block).query('blockquote[0]').setEntry('Pagination', null); + expect(mdq(out).query('blockquote[0]').entries()).toEqual({ container: "'.old'" }); + }); +}); From 812b87594eab7d72dd073fa22a2e72b744c6fb5a Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:32:12 +0300 Subject: [PATCH 13/21] feat(mdq): jq-like CLI runMdq returns { output, code } rather than writing to stdout or exiting, so the whole surface is testable; bin/mdq.ts is the only place that touches the process. Exit codes compose like grep: 0 match, 1 no match, 2 usage or selector error. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- bin/mdq.ts | 14 +++++ package.json | 3 +- src/utils/mdq/cli.ts | 115 +++++++++++++++++++++++++++++++++++++ tests/unit/mdq/cli.test.ts | 99 +++++++++++++++++++++++++++++++ 4 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 bin/mdq.ts create mode 100644 src/utils/mdq/cli.ts create mode 100644 tests/unit/mdq/cli.test.ts diff --git a/bin/mdq.ts b/bin/mdq.ts new file mode 100644 index 00000000..e4309b01 --- /dev/null +++ b/bin/mdq.ts @@ -0,0 +1,14 @@ +#!/usr/bin/env bun +import { readFileSync } from 'node:fs'; +import { runMdq } from '../src/utils/mdq/cli.ts'; + +let stdin = ''; +if (!process.stdin.isTTY) stdin = readFileSync(0, 'utf8'); + +const result = await runMdq(process.argv.slice(2), stdin); +if (result.output) { + let text = result.output; + if (!text.endsWith('\n')) text = `${text}\n`; + process.stdout.write(text); +} +process.exit(result.code); diff --git a/package.json b/package.json index e122d668..a1e7fac5 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,8 @@ }, "bin": { "explorbot": "./dist/bin/explorbot-cli.js", - "prima": "./dist/boat/prima/bin/prima-cli.js" + "prima": "./dist/boat/prima/bin/prima-cli.js", + "mdq": "./dist/bin/mdq.js" }, "files": [ "dist/", diff --git a/src/utils/mdq/cli.ts b/src/utils/mdq/cli.ts new file mode 100644 index 00000000..a253bc0d --- /dev/null +++ b/src/utils/mdq/cli.ts @@ -0,0 +1,115 @@ +import { readFileSync, writeFileSync } from 'node:fs'; +import { Command } from 'commander'; +import { insertAt } from './edit.ts'; +import { MdqError, type MarkdownDoc, type Selection, mdq } from './query.ts'; + +const EDIT_FLAGS = ['remove', 'replace', 'insertBefore', 'insertAfter', 'prepend', 'append', 'addRow', 'addItem', 'set'] as const; + +export async function runMdq(argv: string[], stdin: string): Promise { + const program = new Command(); + program + .name('mdq') + .description('query and edit markdown') + .argument('[selector]', 'markdown selector') + .argument('[file]', 'file to read; stdin when omitted') + .option('-j, --json', 'output rows as JSON') + .option('-c, --count', 'print the number of matches') + .option('-t, --text', 'print unwrapped text') + .option('--frontmatter', 'print frontmatter as JSON') + .option('-i, --in-place', 'write the result back to the file') + .option('--remove', 'delete matched blocks') + .option('--replace ', 'replace matched blocks') + .option('--insert-before ', 'insert before each match') + .option('--insert-after ', 'insert after each match') + .option('--prepend ', 'insert at the start of each match') + .option('--append ', 'insert at the end of each match') + .option('--add-row ', 'append a table row') + .option('--add-item ', 'append a list item') + .option('--set ', 'set an entry; omit the value to delete it') + .exitOverride() + .configureOutput({ writeOut: () => {}, writeErr: () => {} }); + + try { + program.parse(argv, { from: 'user' }); + } catch (error) { + return { output: String((error as Error).message), code: 2 }; + } + + const options = program.opts(); + let [selector, file] = program.args; + if (options.frontmatter && selector && !file) { + file = selector; + selector = ''; + } + + let source = stdin; + if (file) { + try { + source = readFileSync(file, 'utf8'); + } catch { + return { output: `Cannot read ${file}`, code: 2 }; + } + } + + try { + return await apply(mdq(source), selector, options, file); + } catch (error) { + if (error instanceof MdqError) return { output: error.message, code: 2 }; + throw error; + } +} + +async function apply(doc: MarkdownDoc, selector: string, options: Record, file?: string): Promise { + if (options.frontmatter) return { output: `${JSON.stringify(doc.frontmatter(), null, 2)}\n`, code: 0 }; + if (!selector) return { output: 'A selector is required', code: 2 }; + + const chosen = EDIT_FLAGS.filter((flag) => options[flag] !== undefined); + if (chosen.length > 1) return { output: 'Only one edit at a time', code: 2 }; + + const selection = doc.query(selector); + if (chosen.length === 0) return read(selection, options); + + if (!selection.exists()) return finish(String(doc), 1, options, file); + return finish(String(edit(selection, chosen[0], options)), 0, options, file); +} + +function read(selection: Selection, options: Record): CliResult { + const code = selection.exists() ? 0 : 1; + if (options.count) return { output: `${selection.count()}\n`, code: 0 }; + if (options.json) return { output: `${JSON.stringify(selection.rows(), null, 2)}\n`, code }; + if (options.text) + return { + output: selection + .nodes() + .map((node) => node.text) + .join('\n\n'), + code, + }; + return { output: selection.text(), code }; +} + +function edit(selection: Selection, flag: string, options: Record): MarkdownDoc { + if (flag === 'remove') return selection.remove(); + if (flag === 'replace') return selection.replace(options.replace); + if (flag === 'insertBefore') return selection.insertBefore(options.insertBefore); + if (flag === 'insertAfter') return selection.insertAfter(options.insertAfter); + if (flag === 'prepend') return selection.prepend(options.prepend); + if (flag === 'append') return selection.append(options.append); + if (flag === 'addItem') return selection.addItem(options.addItem); + if (flag === 'addRow') return selection.addRow(JSON.parse(options.addRow)); + const separator = options.set.indexOf('='); + if (separator < 0) return selection.setEntry(options.set, null); + return selection.setEntry(options.set.slice(0, separator), options.set.slice(separator + 1) || null); +} + +function finish(output: string, code: number, options: Record, file?: string): CliResult { + if (!options.inPlace) return { output, code }; + if (!file) return { output: '--in-place needs a file', code: 2 }; + writeFileSync(file, output); + return { output: '', code }; +} + +export interface CliResult { + output: string; + code: number; +} diff --git a/tests/unit/mdq/cli.test.ts b/tests/unit/mdq/cli.test.ts new file mode 100644 index 00000000..262a6ee3 --- /dev/null +++ b/tests/unit/mdq/cli.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest'; +import { runMdq } from '../../../src/utils/mdq/cli.ts'; + +const doc = `# Title + +## API + +| Method | Path | +|--------|------| +| GET | /users | + +## FAQ + +question? +`; + +describe('reads', () => { + it('prints matched markdown', async () => { + const result = await runMdq(['h2'], doc); + expect(result.output).toContain('## API'); + expect(result.code).toBe(0); + }); + + it('accepts a leading dot like jq', async () => { + expect((await runMdq(['.h2'], doc)).output).toContain('## API'); + }); + + it('prints rows as json', async () => { + const result = await runMdq(['section("API") table', '--json'], doc); + expect(JSON.parse(result.output)).toEqual([{ Method: 'GET', Path: '/users' }]); + }); + + it('prints a count', async () => { + expect((await runMdq(['h2', '--count'], doc)).output.trim()).toBe('2'); + }); + + it('prints unwrapped text', async () => { + expect((await runMdq(['h2', '--text'], doc)).output).not.toContain('##'); + }); + + it('prints frontmatter as json', async () => { + const result = await runMdq(['--frontmatter'], '---\nurl: /x\n---\n\n# T\n'); + expect(JSON.parse(result.output)).toEqual({ url: '/x' }); + }); +}); + +describe('edits', () => { + it('removes and prints the whole document', async () => { + const result = await runMdq(['section("FAQ")', '--remove'], doc); + expect(result.output).not.toContain('## FAQ'); + expect(result.output).toContain('## API'); + }); + + it('appends into a section', async () => { + expect((await runMdq(['section("FAQ")', '--append', 'answer!'], doc)).output).toContain('answer!'); + }); + + it('adds a table row from json', async () => { + expect((await runMdq(['table', '--add-row', '{"Method":"POST","Path":"/s"}'], doc)).output).toContain('POST'); + }); + + it('sets an entry', async () => { + const result = await runMdq(['blockquote', '--set', 'Container=.x'], '> Container: .old\n'); + expect(result.output).toContain('.x'); + expect(result.output).not.toContain('.old'); + }); + + it('refuses more than one edit at a time', async () => { + const result = await runMdq(['h2', '--remove', '--append', 'x'], doc); + expect(result.code).toBe(2); + expect(result.output).toContain('Only one edit'); + }); +}); + +describe('exit codes', () => { + it('returns 1 when nothing matches', async () => { + expect((await runMdq(['h5'], doc)).code).toBe(1); + }); + + it('returns 2 on an unknown selector', async () => { + const result = await runMdq(['secton("A")'], doc); + expect(result.code).toBe(2); + expect(result.output).toContain('Unknown selector'); + }); + + it('returns 1 when an edit matched nothing, leaving the document intact', async () => { + const result = await runMdq(['h5', '--remove'], doc); + expect(result.code).toBe(1); + expect(result.output).toBe(doc); + }); + + it('returns 0 when an edit matched', async () => { + expect((await runMdq(['h2', '--remove'], doc)).code).toBe(0); + }); + + it('returns 2 without a selector', async () => { + expect((await runMdq([], doc)).code).toBe(2); + }); +}); From c076c99111ea87687941f91028dfe28fc85f70e1 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:33:16 +0300 Subject: [PATCH 14/21] docs(mdq): package README Every example executed against the implementation before committing. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- src/utils/mdq/README.md | 149 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 src/utils/mdq/README.md diff --git a/src/utils/mdq/README.md b/src/utils/mdq/README.md new file mode 100644 index 00000000..3227e86b --- /dev/null +++ b/src/utils/mdq/README.md @@ -0,0 +1,149 @@ +# mdq + +Query and edit markdown with a selector language — jq, for markdown. + +```js +import { mdq } from 'mdq'; + +mdq(readme).query('section("Install") code[0]').text(); +mdq(plan).comment(/^test/).nodes(); +mdq(doc).table().addRow({ Method: 'POST', Path: '/sessions' }).toString(); +``` + +There is a CLI too: + +```bash +mdq 'section("API") table' --json README.md +``` + +## The one rule + +**Reads narrow, writes return the document.** + +`mdq(source)` gives a `MarkdownDoc`. `query()` and the sugar methods narrow it to a +`Selection`. Every write returns a `MarkdownDoc` again, so edits chain and end with +`toString()`: + +```js +mdq(source) + .query('section("API")') + .append('## Notes\n') + .query('blockquote[0]') + .remove() + .toString(); +``` + +## Selectors + +| Selector | Matches | +| --- | --- | +| `section` | a heading and everything under it, until the next heading of the same or shallower depth | +| `section1` … `section6` | the same, restricted to one heading depth | +| `heading`, `h1` … `h6` | the heading line alone | +| `paragraph` | a paragraph | +| `table` | a GFM table | +| `list` | a bullet or ordered list | +| `item` | one item of a list | +| `code` | a fenced code block | +| `blockquote` | a `>` block | +| `hr` | a thematic break | +| `html` | an HTML block, matched on its raw text | +| `comment` | an HTML comment, matched on its **inner** body | + +Text matchers go in parentheses, and `!` negates any of them: + +``` +section("Install") exact +section(~"Inst") contains +section(/^inst/i) regex, with its own flags +section(!~"Draft") negated +``` + +Index and slice with brackets, and compose with spaces to scope one selector inside +another: + +``` +heading[0] first +heading[-1] last +blockquote[2:5] a slice +section("API") table every table inside that section +``` + +A leading `.` is accepted and ignored, so `.h2` works if that is your habit from jq. + +An unknown selector throws `MdqSelectorError`, which carries the `index` of the offending +character. It never silently matches nothing. + +## Matchers as values + +Passing a JavaScript value avoids escaping a dynamic string into a selector: + +```js +mdq(doc).query('section2', section.name); // exact +mdq(doc).heading(/^summary/i); // regex, own flags +mdq(doc).item((text) => text.length > 80); // predicate +``` + +A `string` matches exactly, a `RegExp` honors its own flags, and a function is a predicate +over the node's text. Every sugar method takes one: `section` `heading` `paragraph` `table` +`list` `item` `code` `blockquote` `comment` `html` `hr`. Each is exactly +`query(selector, matcher)`; `section` and `heading` also take `{ depth }`. + +## Reading + +| Method | Returns | +| --- | --- | +| `text()` | raw markdown of every match, joined | +| `nodes()` | `{ type, depth, text }` per match | +| `rows()` | table rows as objects, keyed by header | +| `entries()` | `Key: value` lines of a block, keys lowercased | +| `count()` / `exists()` | how many matched / whether any did | +| `first()` / `last()` / `at(n)` / `slice(from, to)` | narrow the selection | +| `each()` | one single-match `Selection` per match | +| `preceding()` / `following()` | everything before the first / after the last match | + +## Writing + +Every one returns a `MarkdownDoc`. + +| Method | Effect | +| --- | --- | +| `replace(md)` | replace each match | +| `replaceEach(fn)` | replace each match with `fn(selection, index)` | +| `remove()` | delete each match, and its blank line | +| `insertBefore(md)` / `insertAfter(md)` | add a sibling block | +| `prepend(md)` / `append(md)` | add a block inside a section or list | +| `addRow(obj)` | append a table row, re-aligning the columns | +| `addItem(text)` | append a list item, copying the existing marker | +| `setEntry(key, value)` | set a `Key: value` line; `null` deletes it | + +`MarkdownDoc` itself has `append` and `prepend`, which act on the whole document. + +Anything that takes markdown also takes a `MarkdownDoc`, so fragments compose without a +`toString()` hop. Writes never leave zero blank lines between blocks, and never more than +one — including inside fenced code blocks, which are left exactly as they are. + +## Frontmatter + +A leading `---` block is parsed as YAML, kept out of the token index, and exposed as data. +Without this, `marked` reads `url: /login` as a setext heading. + +```js +const doc = mdq(page); +doc.frontmatter(); // { url: '/login', wait: 1000, tags: ['auth'] } +doc.query('h2').count(); // 0 — the --- block is not a heading +doc.setFrontmatter('wait', 2000); // comments and formatting survive +``` + +## Limitations + +- **Block-level comments only.** A comment inside a paragraph (`text more`) is + part of that paragraph's token and is not reachable as a `comment`. +- **No row or item selectors.** `addRow` and `addItem` append; there is no `removeRow`, + because there is nothing to select. +- **YAML frontmatter only.** TOML (`+++`) and JSON blocks are skipped from the token index + but not parsed. + +## Dependencies + +`marked` and `yaml`. Nothing else. From ab2495ecf07e0408c2f112af94f34b83e347284d Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:36:09 +0300 Subject: [PATCH 15/21] docs: changelog for mdq package and CLI Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- CHANGELOG.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f1df0ae..80592755 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,59 @@ # Changelog +## 2026-09-15 + +### New CLI Options + +- **`mdq`** — A new command for reading and editing markdown from the shell, the way `jq` reads JSON. + The first argument is a selector, the second an optional file (stdin is used when it is omitted). + Matched markdown is printed by default. Exit codes compose like `grep`: `0` when something matched, + `1` when nothing did, `2` for a bad selector or bad usage. + + ```bash + mdq 'h2' README.md # print every h2 + cat plan.md | mdq 'section("API") table' # read from stdin + mdq 'comment(~"test")' plan.md --count # how many test comments + ``` + +- **`--json`, `--count`, `--text`, `--frontmatter`** — Change what is printed: table rows as JSON, + the number of matches, the text with its markdown stripped, or the file's frontmatter as JSON. + + ```bash + mdq 'section("API") table' --json README.md + mdq 'h2' --count README.md + mdq --frontmatter knowledge/login.md + ``` + +- **`--remove`, `--replace`, `--insert-before`, `--insert-after`, `--prepend`, `--append`, `--add-row`, `--add-item`, `--set`** — + Edit the file instead of reading it. The whole document is printed with the edit applied; one edit at a + time. `--add-row` takes a JSON object and lines the table's columns back up; `--add-item` copies the + list's existing bullet or numbering; `--set` takes `Key=value` and drops the entry when the value is empty. + + ```bash + mdq 'section("FAQ")' doc.md --remove + mdq 'table[0]' api.md --add-row '{"Method":"GET","Path":"/users"}' + mdq 'list' plan.md --add-item 'check the dashboard' + ``` + +- **`-i`, `--in-place`** — Write the edit back to the file instead of printing it. + + ```bash + mdq 'section("Draft")' notes.md --remove -i + ``` + +### Changes + +- Knowledge and experience files are now read correctly when they start with a `---` block. The + frontmatter used to be read as a heading titled `url: /login`, so the first fact in every such file + could be mistaken for a section title. +- Blank lines are no longer lost or doubled when a block is deleted or inserted, and blank lines inside + fenced code blocks are left exactly as they were. +- A mistyped selector now fails with a message naming the unknown word, instead of quietly matching + nothing and looking like an empty file. +- A search written as `/pattern/` is now case-sensitive unless it ends with `i`, matching how quoted + searches already behaved. Previously every `/pattern/` ignored case whether it said so or not. +- Searching a table now looks at its cells, not only its column titles. + ## 2026-09-11 ### Changes From 6b22aa6814c904f454e6bf0b5f3b9063000ca7e7 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:38:32 +0300 Subject: [PATCH 16/21] test(mdq): pin how a Selection stringifies, and document it Selection extends MarkdownDoc to share the sugar layer, so it type-checks anywhere a MarkdownDoc does, but stringifies to its matched markdown rather than the whole document. That is the useful behaviour - a selection can be passed straight back as a fragment - but it was previously unpinned by any test and undocumented. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- .../plans/2026-09-15-mdq-package.md | 138 +++++++++--------- src/utils/mdq/README.md | 10 ++ tests/unit/mdq/sugar.test.ts | 22 +++ 3 files changed, 101 insertions(+), 69 deletions(-) diff --git a/docs/superpowers/plans/2026-09-15-mdq-package.md b/docs/superpowers/plans/2026-09-15-mdq-package.md index ac7aed23..fc4a0ac8 100644 --- a/docs/superpowers/plans/2026-09-15-mdq-package.md +++ b/docs/superpowers/plans/2026-09-15-mdq-package.md @@ -1,6 +1,6 @@ # mdq Package Implementation Plan -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [x]`) syntax for tracking. **Goal:** Extract `src/utils/markdown-query.ts` into `src/utils/mdq/` as a publish-ready package that can both query and update markdown, then add a jq-like CLI. @@ -68,7 +68,7 @@ Move the parser to its new home and teach it the one thing it gets wrong today: The `trailing` field is new and load-bearing: `marked` emits `space` tokens as siblings (a `paragraph` raw is `"para"` with no newline, followed by a separate `space` raw of `"\n\n"`), so every write verb needs to know where a node's separator lives. -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // tests/unit/mdq/frontmatter.test.ts @@ -127,12 +127,12 @@ describe('buildTokenIndex', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/frontmatter.test.ts` Expected: FAIL — cannot resolve `src/utils/mdq/query.ts` -- [ ] **Step 3: Create query.ts with the index** +- [x] **Step 3: Create query.ts with the index** Copy `src/utils/markdown-query.ts` to `src/utils/mdq/query.ts` verbatim first, then apply these three changes. @@ -207,12 +207,12 @@ export interface MatchedRange { } ``` -- [ ] **Step 4: Run test to verify it passes** +- [x] **Step 4: Run test to verify it passes** Run: `bun test tests/unit/mdq/frontmatter.test.ts` Expected: PASS (8 tests) -- [ ] **Step 5: Format, lint and commit** +- [x] **Step 5: Format, lint and commit** ```bash bun run format && bun run lint:fix @@ -236,7 +236,7 @@ Get every existing call site running against the new file with **zero behaviour - Consumes: `buildTokenIndex`, `splitFrontmatter`, `MatchedRange` from Task 1 - Produces: `mdq(source: string): MarkdownQuery`, class `MarkdownQuery`, `parseQuery`, all existing methods unchanged -- [ ] **Step 1: Move the test file and repoint its import** +- [x] **Step 1: Move the test file and repoint its import** ```bash git mv tests/unit/markdown-query.test.ts tests/unit/mdq/query.test.ts @@ -254,12 +254,12 @@ to: import { mdq, parseQuery } from '../../../src/utils/mdq/query.ts'; ``` -- [ ] **Step 2: Run the suite to verify it fails** +- [x] **Step 2: Run the suite to verify it fails** Run: `bun test tests/unit/mdq/query.test.ts` Expected: FAIL — `mdq` / `parseQuery` are not yet exported from `query.ts`, or section tests fail because `expandSectionRanges` still assumes `space` tokens are present -- [ ] **Step 3: Restore the full API in query.ts** +- [x] **Step 3: Restore the full API in query.ts** Everything from the original `markdown-query.ts` below `buildTokenIndex` — `matchText`, `entryKey`, `getTokenText`, `getHeadingDepth`, `isSectionSelector`, `getSectionDepth`, `selectorToTokenType`, `computeSections`, `extractListItems`, `applyIndexSlice`, `expandSectionRanges`, `executeSegments`, `class MarkdownQuery`, `mdq` — carries over unchanged, except: @@ -277,23 +277,23 @@ for (let j = i + 1; j < candidates.length; j++) { Move every `export interface` / `export type` to the end of the file, and replace the ternaries at the original lines 30, 34, 129-130 and 295 with early returns. -- [ ] **Step 4: Run the suite to verify it passes** +- [x] **Step 4: Run the suite to verify it passes** Run: `bun test tests/unit/mdq/query.test.ts` Expected: PASS — **110 tests**, the same count as before the move -- [ ] **Step 5: Replace markdown-query.ts with a shim** +- [x] **Step 5: Replace markdown-query.ts with a shim** ```ts export * from './mdq/query.ts'; ``` -- [ ] **Step 6: Verify every existing call site still works** +- [x] **Step 6: Verify every existing call site still works** Run: `bun test tests/unit/` Expected: PASS, no new failures versus the pre-task run -- [ ] **Step 7: Format, lint and commit** +- [x] **Step 7: Format, lint and commit** ```bash bun run format && bun run lint:fix @@ -327,7 +327,7 @@ The one risky task. It ends with the repo green and every break fixed. - `mdq(source: Markdown): MarkdownDoc` - Deprecated alias `MarkdownQuery = Selection` -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // append to tests/unit/mdq/query.test.ts @@ -359,12 +359,12 @@ describe('MarkdownDoc chaining', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/query.test.ts -t 'MarkdownDoc chaining'` Expected: FAIL — `.query is not a function` on the string returned by `replace` -- [ ] **Step 3: Split MarkdownQuery into MarkdownDoc and Selection** +- [x] **Step 3: Split MarkdownQuery into MarkdownDoc and Selection** `MarkdownDoc` holds the source. `Selection` holds source plus matches. Every write on `Selection` ends by wrapping its result: @@ -431,12 +431,12 @@ git grep -ln "MarkdownQuery" -- src bin boat tests Expected: only `src/utils/mdq/query.ts`. Any other file needs its import checked. -- [ ] **Step 4: Run test to verify it passes** +- [x] **Step 4: Run test to verify it passes** Run: `bun test tests/unit/mdq/query.test.ts -t 'MarkdownDoc chaining'` Expected: PASS (4 tests) -- [ ] **Step 5: Fix the write assertions in the ported suite** +- [x] **Step 5: Fix the write assertions in the ported suite** The existing `replace` tests assert against a string. Wrap each in `String(...)`, for example: @@ -453,7 +453,7 @@ Apply the same to every assertion in the `replace`, `setKeyValue` and `edge case Run: `bun test tests/unit/mdq/query.test.ts` Expected: PASS — 114 tests -- [ ] **Step 6: Migrate the four breakage classes** +- [x] **Step 6: Migrate the four breakage classes** **(a) Assignment into a `string`-typed target** — append `.toString()`: @@ -501,7 +501,7 @@ Expected: PASS — 114 tests **Leave alone** — these already work because `mdq()` accepts a `MarkdownDoc`: `planner.ts:303`, `planner.ts:405`. -- [ ] **Step 7: Fix the one regex call site** +- [x] **Step 7: Fix the one regex call site** `src/ai/researcher.ts:316` relies on regex matching being implicitly case-insensitive. Task 4 removes that. Make the flag explicit now so the two changes never overlap: @@ -509,7 +509,7 @@ Expected: PASS — 114 tests const summaryText = mdq(result.text).query('section2(/^summary/i)').query('paragraph[0]').text().trim(); ``` -- [ ] **Step 8: Verify with the scoped type check** +- [x] **Step 8: Verify with the scoped type check** Grep by **type name, repo-wide** — not by file path. A path-scoped grep misses test helpers and bin scripts, and those break too: @@ -530,12 +530,12 @@ the same shape — a function whose declared return type is `string` now returns which does **not** coerce and throws `e.replace is not a function` at runtime. Trust the grep, not the list. -- [ ] **Step 9: Run the full unit suite** +- [x] **Step 9: Run the full unit suite** Run: `bun test tests/unit/` Expected: PASS, no new failures -- [ ] **Step 10: Format, lint and commit** +- [x] **Step 10: Format, lint and commit** ```bash bun run format && bun run lint:fix @@ -557,7 +557,7 @@ Four grammar changes, all additive now that Task 3 pre-fixed the one regex call - Consumes: `parseQuery`, `getTokenText`, `selectorToTokenType`, `matchText` from Task 2 - Produces: `class MdqError extends Error`, `class MdqSelectorError extends MdqError` (with `index: number`), selectors `comment` and `html` -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // tests/unit/mdq/selectors.test.ts @@ -656,12 +656,12 @@ describe('selector errors', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/selectors.test.ts` Expected: FAIL — `MdqSelectorError` is not exported -- [ ] **Step 3: Implement the four changes** +- [x] **Step 3: Implement the four changes** Error classes, at the top of the class section: @@ -786,17 +786,17 @@ export interface NodeInfo { } ``` -- [ ] **Step 4: Run test to verify it passes** +- [x] **Step 4: Run test to verify it passes** Run: `bun test tests/unit/mdq/selectors.test.ts` Expected: PASS (13 tests) -- [ ] **Step 5: Verify nothing regressed** +- [x] **Step 5: Verify nothing regressed** Run: `bun test tests/unit/` Expected: PASS — in particular `query.test.ts` still at 114, since Task 3 already fixed `researcher.ts:316` -- [ ] **Step 6: Format, lint and commit** +- [x] **Step 6: Format, lint and commit** ```bash bun run format && bun run lint:fix @@ -823,7 +823,7 @@ Removes the hand-escaping wart: `section.name.replace(/"/g, '\\"')` at `research - Sugar on both classes: `section` `heading` `paragraph` `table` `list` `item` `code` `blockquote` `comment` `html` `hr` - `at(n: number): Selection` and `slice(from?: number, to?: number): Selection` on `Selection` -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // tests/unit/mdq/sugar.test.ts @@ -935,12 +935,12 @@ describe('canonical read names', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/sugar.test.ts` Expected: FAIL — `mdq(...).heading is not a function` -- [ ] **Step 3: Implement matchers and the shared sugar base** +- [x] **Step 3: Implement matchers and the shared sugar base** A value matcher bypasses the grammar entirely, so it needs its own `TextMatcher` mode: @@ -1044,7 +1044,7 @@ On `Selection`, add: At the end of the file add `Matcher`, `SelectorOptions`, `predicate?: (text: string) => boolean` on `TextMatcher`, and widen its `mode` to include `'predicate'`. -- [ ] **Step 4: Run test to verify it passes** +- [x] **Step 4: Run test to verify it passes** Run: `bun test tests/unit/mdq/sugar.test.ts` Expected: PASS (21 tests) @@ -1052,7 +1052,7 @@ Expected: PASS (21 tests) The `canonical read names` block fails until Step 5 adds the renames — that is expected. Run Step 5 before treating those four as real failures. -- [ ] **Step 5: Add the read renames and their deprecated aliases** +- [x] **Step 5: Add the read renames and their deprecated aliases** Canonical names, with the old ones kept and marked: @@ -1077,7 +1077,7 @@ Each alias is one line, for example: Run: `bun test tests/unit/` Expected: PASS — the ported suite still calls the deprecated names and must keep working -- [ ] **Step 6: Format, lint and commit** +- [x] **Step 6: Format, lint and commit** ```bash bun run format && bun run lint:fix @@ -1110,7 +1110,7 @@ This matters because `marked` separators are uneven: a `heading` raw is `"# A\n\ - Produces, on `Selection`: `remove()`, `insertBefore(md)`, `insertAfter(md)`, `prepend(md)`, `append(md)` - Produces, on `MarkdownDoc`: `append(md)`, `prepend(md)` -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // tests/unit/mdq/edit.test.ts @@ -1216,12 +1216,12 @@ describe('chained edits', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/edit.test.ts` Expected: FAIL — `.remove is not a function` -- [ ] **Step 3: Write edit.ts** +- [x] **Step 3: Write edit.ts** ```ts import type { MatchedRange } from './query.ts'; @@ -1310,7 +1310,7 @@ raw carries its own `"\n\n"`. So "no trailing space, therefore trim backwards" i `[start, blockEnd)` is already correct whenever anything follows; only a node removed from the very end needs repair. -- [ ] **Step 4: Wire the verbs onto Selection and MarkdownDoc** +- [x] **Step 4: Wire the verbs onto Selection and MarkdownDoc** On `Selection`, four public verbs delegating to one private helper: @@ -1389,14 +1389,14 @@ export class MdqOperationError extends MdqError { } ``` -- [ ] **Step 5: Run test to verify it passes** +- [x] **Step 5: Run test to verify it passes** Run: `bun test tests/unit/mdq/edit.test.ts` Expected: PASS (19 tests) If a whitespace assertion fails, print the actual output with `JSON.stringify` before changing anything — the difference is almost always one newline, and guessing at it will break a different case. -- [ ] **Step 6: Run the whole suite and commit** +- [x] **Step 6: Run the whole suite and commit** ```bash bun test tests/unit/ @@ -1419,7 +1419,7 @@ git commit -m "feat(mdq): remove and insert verbs with whitespace normalization" - Produces in `edit.ts`: `renderTable(headers: string[], rows: string[][], align: (string | null)[]): string`, `renderItem(listRaw: string, text: string): string` - Produces on `Selection`: `addRow(row: Record): MarkdownDoc`, `addItem(text: string): MarkdownDoc` -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // tests/unit/mdq/structural.test.ts @@ -1485,12 +1485,12 @@ describe('addItem', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/structural.test.ts` Expected: FAIL — `.addRow is not a function` -- [ ] **Step 3: Add the renderers to edit.ts** +- [x] **Step 3: Add the renderers to edit.ts** ```ts export function renderTable(headers: string[], rows: string[][], align: (string | null)[]): string { @@ -1518,7 +1518,7 @@ function dashes(alignment: string | null, width: number): string { } ``` -- [ ] **Step 4: Wire the verbs onto Selection** +- [x] **Step 4: Wire the verbs onto Selection** ```ts addRow(row: Record): MarkdownDoc { @@ -1544,12 +1544,12 @@ function dashes(alignment: string | null, width: number): string { } ``` -- [ ] **Step 5: Run test to verify it passes** +- [x] **Step 5: Run test to verify it passes** Run: `bun test tests/unit/mdq/structural.test.ts` Expected: PASS (10 tests) -- [ ] **Step 6: Run the whole suite and commit** +- [x] **Step 6: Run the whole suite and commit** ```bash bun test tests/unit/ @@ -1575,7 +1575,7 @@ git commit -m "feat(mdq): addRow and addItem structural inserts" Reading and writing both go through `yaml`'s **Document API** (`YAML.parseDocument`), never `parse`/`stringify`. That is what preserves comments through a write — verified behaviour, not an assumption. -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // append to tests/unit/mdq/frontmatter.test.ts @@ -1646,12 +1646,12 @@ describe('entries and setEntry', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/frontmatter.test.ts` Expected: FAIL — `.frontmatter is not a function` -- [ ] **Step 3: Implement in edit.ts** +- [x] **Step 3: Implement in edit.ts** ```ts import YAML from 'yaml'; @@ -1696,7 +1696,7 @@ export function rewriteEntries(tokenText: string, isBlockquote: boolean, key: st `entryKey` moves to `edit.ts` alongside it. -- [ ] **Step 4: Wire onto the classes** +- [x] **Step 4: Wire onto the classes** ```ts frontmatter(): Record { @@ -1719,12 +1719,12 @@ export function rewriteEntries(tokenText: string, isBlockquote: boolean, key: st } ``` -- [ ] **Step 5: Run test to verify it passes** +- [x] **Step 5: Run test to verify it passes** Run: `bun test tests/unit/mdq/frontmatter.test.ts` Expected: PASS (21 tests) -- [ ] **Step 6: Confirm knowledge and experience files now parse correctly** +- [x] **Step 6: Confirm knowledge and experience files now parse correctly** This is the real-world check that motivated the feature: @@ -1744,7 +1744,7 @@ Expected: frontmatter parsed as an object on each file, and **no heading whose t If either directory is empty, skip this step and note it. -- [ ] **Step 7: Run the whole suite and commit** +- [x] **Step 7: Run the whole suite and commit** ```bash bun test tests/unit/ @@ -1766,7 +1766,7 @@ The package is publish-ready only if someone can use it without reading the sour - Consumes: the complete API from Tasks 3-8 - Produces: nothing code depends on -- [ ] **Step 1: Write the README** +- [x] **Step 1: Write the README** Cover, in this order: @@ -1784,7 +1784,7 @@ Do **not** document the deprecated aliases (`get` `toJson` `keyValue` `setKeyVal Follow the repo docs style: show each format example once, and do not close with a "Why this matters" section. -- [ ] **Step 2: Verify every example in the README actually runs** +- [x] **Step 2: Verify every example in the README actually runs** Extract each fenced `js` block and execute it. Any example that throws or prints something other than what the README claims is a documentation bug — fix the README, not the test. @@ -1795,7 +1795,7 @@ import { mdq } from "./src/utils/mdq/query.ts"; ' ``` -- [ ] **Step 3: Commit** +- [x] **Step 3: Commit** ```bash git add src/utils/mdq/README.md @@ -1820,7 +1820,7 @@ git commit -m "docs(mdq): package README" Note a deliberate deviation from `CLAUDE.md`: command logic normally lives in `src/commands/`, but mdq must not import from anywhere in explorbot. Its CLI ships with the package. -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // tests/unit/mdq/cli.test.ts @@ -1910,12 +1910,12 @@ describe('exit codes', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/cli.test.ts` Expected: FAIL — cannot resolve `cli.ts` -- [ ] **Step 3: Implement cli.ts** +- [x] **Step 3: Implement cli.ts** Use Commander with `exitOverride()` and `.configureOutput()` so a parse failure surfaces as a return value rather than killing the process. Shape: @@ -1962,12 +1962,12 @@ export interface CliResult { } ``` -- [ ] **Step 4: Run test to verify it passes** +- [x] **Step 4: Run test to verify it passes** Run: `bun test tests/unit/mdq/cli.test.ts` Expected: PASS (13 tests) -- [ ] **Step 5: Add the thin bin entry** +- [x] **Step 5: Add the thin bin entry** ```ts #!/usr/bin/env bun @@ -1986,7 +1986,7 @@ Add to `package.json` `bin`: "mdq": "./dist/bin/mdq.js" ``` -- [ ] **Step 6: Smoke-test the real binary** +- [x] **Step 6: Smoke-test the real binary** ```bash echo '# A @@ -2007,7 +2007,7 @@ bun run bin/mdq.ts 'nonsense' CLAUDE.md; echo "exit=$?" ``` Expected: `Unknown selector "nonsense"` and `exit=2` -- [ ] **Step 7: Run the whole suite and commit** +- [x] **Step 7: Run the whole suite and commit** ```bash bun test tests/unit/ @@ -2023,12 +2023,12 @@ git commit -m "feat(mdq): jq-like CLI" **Files:** - Modify: `CHANGELOG.md` -- [ ] **Step 1: Run the full unit suite** +- [x] **Step 1: Run the full unit suite** Run: `bun test tests/unit/` Expected: PASS, no failures -- [ ] **Step 2: Re-run the scoped type check** +- [x] **Step 2: Re-run the scoped type check** ```bash bunx tsc -p tsconfig.json --noEmit 2>&1 | grep -E "^(src/utils/mdq/|src/utils/markdown-query|src/experience-tracker|src/ai/planner|src/ai/researcher|bin/mdq)" @@ -2036,7 +2036,7 @@ bunx tsc -p tsconfig.json --noEmit 2>&1 | grep -E "^(src/utils/mdq/|src/utils/ma Expected: **exactly the two known `locators.ts(247,...)` lines.** Anything else is a real defect that CI will not catch. -- [ ] **Step 3: Confirm the package has no explorbot imports** +- [x] **Step 3: Confirm the package has no explorbot imports** ```bash grep -rn "^import\|from '" src/utils/mdq/*.ts | grep -v "'marked'" | grep -v "'yaml'" | grep -v "'commander'" | grep -v "'./" @@ -2044,11 +2044,11 @@ grep -rn "^import\|from '" src/utils/mdq/*.ts | grep -v "'marked'" | grep -v "'y Expected: **no output.** Any line here breaks extractability, which is the whole point of the package. -- [ ] **Step 4: Update the changelog** +- [x] **Step 4: Update the changelog** Use the `/changelog` skill, per `CLAUDE.md`. -- [ ] **Step 5: Commit** +- [x] **Step 5: Commit** ```bash git add CHANGELOG.md diff --git a/src/utils/mdq/README.md b/src/utils/mdq/README.md index 3227e86b..f27d9a3f 100644 --- a/src/utils/mdq/README.md +++ b/src/utils/mdq/README.md @@ -33,6 +33,16 @@ mdq(source) .toString(); ``` +The two stringify differently, and the difference is deliberate: a `MarkdownDoc` gives the +whole document, a `Selection` gives only the markdown it matched. So a selection can be +passed straight back in as a fragment. + +```js +const fragment = mdq(other).query('section("Setup")'); +mdq(doc).query('h2').insertAfter(fragment); // inserts that section +String(mdq(doc).query('paragraph')); // the paragraphs, not the document +``` + ## Selectors | Selector | Matches | diff --git a/tests/unit/mdq/sugar.test.ts b/tests/unit/mdq/sugar.test.ts index 83787993..66829e74 100644 --- a/tests/unit/mdq/sugar.test.ts +++ b/tests/unit/mdq/sugar.test.ts @@ -120,3 +120,25 @@ describe('canonical read names', () => { expect(block.entries()).toEqual(block.keyValue()); }); }); + +describe('stringification', () => { + const src = '# A\n\ntext\n'; + + it('stringifies a document to the whole document', () => { + expect(String(mdq(src))).toBe(src); + }); + + it('stringifies a selection to its matched markdown, not the document', () => { + expect(String(mdq(src).query('paragraph'))).toBe('text\n'); + expect(`${mdq(src).query('paragraph')}`).toBe('text\n'); + }); + + it('treats a selection passed as markdown as the fragment it matched', () => { + const fragment = mdq('### F\n\nfrag\n').query('h3'); + expect(mdq('## A\n\nbody\n').query('h2').insertAfter(fragment).toString()).toBe('## A\n\n### F\n\nbody\n'); + }); + + it('re-opens a selection as a document containing only the match', () => { + expect(mdq(mdq(src).query('paragraph')).toString()).toBe('text\n'); + }); +}); From b379e43a0ce7e12cf98eeacbf89891b1fcf90254 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:49:13 +0300 Subject: [PATCH 17/21] refactor(mdq): finish the query/edit boundary the spec draws The duplication review found two places where query.ts kept logic the spec assigns to edit.ts: - replaceEach reimplemented spliceRanges verbatim, so the one write path used by replace() and setEntry() was the only one not delegating. - setEntry inlined the whole entry-rewrite transformation; it now calls rewriteEntries, matching the shape of every other verb. Also: entryKey moves to edit.ts beside rewriteEntries, dedupeRanges becomes internal now that query.ts no longer needs it, two new ternaries are gone, a dead insertAt import is dropped, three hand-rolled /\s+$/ strips become trimEnd(), and isSectionSelector's /^section\d?$/ now matches the [1-6] every other depth check uses. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- src/utils/mdq/cli.ts | 4 ++-- src/utils/mdq/edit.ts | 32 +++++++++++++++++++++++++++----- src/utils/mdq/query.ts | 33 ++++----------------------------- 3 files changed, 33 insertions(+), 36 deletions(-) diff --git a/src/utils/mdq/cli.ts b/src/utils/mdq/cli.ts index a253bc0d..28140d91 100644 --- a/src/utils/mdq/cli.ts +++ b/src/utils/mdq/cli.ts @@ -1,6 +1,5 @@ import { readFileSync, writeFileSync } from 'node:fs'; import { Command } from 'commander'; -import { insertAt } from './edit.ts'; import { MdqError, type MarkdownDoc, type Selection, mdq } from './query.ts'; const EDIT_FLAGS = ['remove', 'replace', 'insertBefore', 'insertAfter', 'prepend', 'append', 'addRow', 'addItem', 'set'] as const; @@ -74,7 +73,8 @@ async function apply(doc: MarkdownDoc, selector: string, options: Record): CliResult { - const code = selection.exists() ? 0 : 1; + let code = 1; + if (selection.exists()) code = 0; if (options.count) return { output: `${selection.count()}\n`, code: 0 }; if (options.json) return { output: `${JSON.stringify(selection.rows(), null, 2)}\n`, code }; if (options.text) diff --git a/src/utils/mdq/edit.ts b/src/utils/mdq/edit.ts index ce66b352..41db15b1 100644 --- a/src/utils/mdq/edit.ts +++ b/src/utils/mdq/edit.ts @@ -8,7 +8,7 @@ export function splitFrontmatter(source: string): FrontmatterSplit { return { raw: match[1], body: source.slice(match[0].length), offset: match[0].length }; } -export function dedupeRanges(ranges: MatchedRange[]): MatchedRange[] { +function dedupeRanges(ranges: MatchedRange[]): MatchedRange[] { const sorted = [...ranges].sort((a, b) => a.start - b.start); const kept: MatchedRange[] = []; let lastEnd = -1; @@ -26,7 +26,7 @@ export function blockEnd(range: MatchedRange): number { } export function normalizeBlock(markdown: string): string { - return `${markdown.replace(/\s+$/, '')}\n`; + return `${markdown.trimEnd()}\n`; } export function removeRanges(source: string, ranges: MatchedRange[]): string { @@ -44,7 +44,7 @@ export function removeRanges(source: string, ranges: MatchedRange[]): string { result = ''; continue; } - result = `${head.replace(/\n+$/, '')}\n`; + result = `${head.trimEnd()}\n`; } return result; } @@ -93,6 +93,27 @@ function dashes(alignment: string | null, width: number): string { return '-'.repeat(width); } +export function entryKey(line: string): string | null { + const separator = line.indexOf(':'); + if (separator < 1) return null; + return line.slice(0, separator).trim().toLowerCase(); +} + +export function rewriteEntries(text: string, key: string, value: string | null, isBlockquote: boolean): string { + const lines = text + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); + + const index = lines.findIndex((line) => entryKey(line) === key.toLowerCase()); + if (index < 0 && value) lines.push(`${key}: ${value}`); + if (index >= 0 && value) lines[index] = `${key}: ${value}`; + if (index >= 0 && !value) lines.splice(index, 1); + + if (!isBlockquote) return lines.join('\n'); + return lines.map((line) => `> ${line}`).join('\n'); +} + export function readFrontmatter(source: string): Record { const { raw } = splitFrontmatter(source); if (!raw) return {}; @@ -101,10 +122,11 @@ export function readFrontmatter(source: string): Record { export function writeFrontmatter(source: string, key: string, value: unknown): string { const { raw, body, offset } = splitFrontmatter(source); - const document = raw ? YAML.parseDocument(raw) : new YAML.Document({}); + let document = new YAML.Document({}); + if (raw) document = YAML.parseDocument(raw); if (value === null) document.delete(key); if (value !== null) document.set(key, value); - const rendered = document.toString().replace(/\s+$/, ''); + const rendered = document.toString().trimEnd(); if (!offset) return `---\n${rendered}\n---\n\n${source}`; return `---\n${rendered}\n---\n${body}`; } diff --git a/src/utils/mdq/query.ts b/src/utils/mdq/query.ts index 81fe9a00..e5be60ed 100644 --- a/src/utils/mdq/query.ts +++ b/src/utils/mdq/query.ts @@ -1,5 +1,5 @@ import { type Token, type Tokens, marked } from 'marked'; -import { blockEnd, dedupeRanges, insertAt, readFrontmatter, removeRanges, renderItem, renderTable, spliceRanges, splitFrontmatter, writeFrontmatter } from './edit.ts'; +import { blockEnd, entryKey, insertAt, readFrontmatter, removeRanges, renderItem, renderTable, rewriteEntries, spliceRanges, splitFrontmatter, writeFrontmatter } from './edit.ts'; export { splitFrontmatter }; @@ -204,12 +204,6 @@ function matchText(text: string, matcher: TextMatcher): boolean { return result; } -function entryKey(line: string): string | null { - const separator = line.indexOf(':'); - if (separator < 1) return null; - return line.slice(0, separator).trim().toLowerCase(); -} - function getTokenText(token: Token): string { const t = token as any; switch (token.type) { @@ -236,7 +230,7 @@ function getHeadingDepth(selector: string): number | null { } function isSectionSelector(selector: string): boolean { - return /^section\d?$/.test(selector); + return /^section[1-6]?$/.test(selector); } function getSectionDepth(selector: string): number | null { @@ -595,18 +589,7 @@ export class Selection extends MarkdownDoc { setEntry(key: string, value: string | null): MarkdownDoc { return this.replaceEach((match) => { const token = match.matches[0].token; - const lines = getTokenText(token) - .split('\n') - .map((line) => line.trim()) - .filter(Boolean); - - const index = lines.findIndex((line) => entryKey(line) === key.toLowerCase()); - if (index < 0 && value) lines.push(`${key}: ${value}`); - if (index >= 0 && value) lines[index] = `${key}: ${value}`; - if (index >= 0 && !value) lines.splice(index, 1); - - if (token.type !== 'blockquote') return lines.join('\n'); - return lines.map((line) => `> ${line}`).join('\n'); + return rewriteEntries(getTokenText(token), key, value, token.type === 'blockquote'); }); } @@ -662,15 +645,7 @@ export class Selection extends MarkdownDoc { } replaceEach(replacer: (match: Selection, index: number) => Markdown): MarkdownDoc { - const kept = dedupeRanges(this.matches); - const replacements = kept.map((range, index) => String(replacer(new Selection(this.source, [range]), index))); - let result = this.source; - for (let i = kept.length - 1; i >= 0; i--) { - const range = kept[i]; - result = result.slice(0, range.start) + replacements[i] + result.slice(range.start + range.length); - } - - return new MarkdownDoc(result); + return new MarkdownDoc(spliceRanges(this.source, this.matches, (range, index) => String(replacer(new Selection(this.source, [range]), index)))); } count(): number { From 07a0e4499acb560f3c248af432bd9061f9be6417 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Thu, 17 Sep 2026 00:58:22 +0300 Subject: [PATCH 18/21] feat(mdq): publish as a standalone npm package runnable with npx bin/mdq.ts is now Node-compatible, not Bun-only. Two real breaks fixed: - readFileSync(0, 'utf8') throws EAGAIN on Node when stdin is a pipe with no data ready, crashing with a stack trace. Bun tolerates it. Stdin is now read asynchronously. - Stdin was read whenever it was not a TTY, before knowing whether a file argument was given, so 'mdq h2 file.md' in a pipeline read input it never needed. runMdq now takes a lazy reader and calls it only when no file is given. Packaging mirrors prima-cli: src/utils/mdq/package.json declares the standalone manifest, scripts/build-mdq-npm.ts bundles both entries with bun build --target=node (marked, yaml and commander stay external) and emits declarations, and .github/workflows/publish-mdq.yml publishes on a release tagged mdq-*. The workflow's smoke test installs the tarball into a clean project and exercises the CLI, stdin, the EAGAIN case, exit codes and a library import on Node - the same sequence was run locally. The mdq bin is removed from explorbot's package.json so the two packages cannot collide on the binary name. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- .github/workflows/publish-mdq.yml | 97 +++++++++++++++++++++++++++++++ .gitignore | 1 + CHANGELOG.md | 5 ++ bin/mdq.ts | 14 +++-- package.json | 6 +- scripts/build-mdq-npm.ts | 74 +++++++++++++++++++++++ src/utils/mdq/README.md | 16 +++-- src/utils/mdq/cli.ts | 7 ++- src/utils/mdq/package.json | 28 +++++++++ tests/unit/mdq/cli.test.ts | 55 +++++++++++++----- 10 files changed, 273 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/publish-mdq.yml create mode 100644 scripts/build-mdq-npm.ts create mode 100644 src/utils/mdq/package.json diff --git a/.github/workflows/publish-mdq.yml b/.github/workflows/publish-mdq.yml new file mode 100644 index 00000000..5abd5fc8 --- /dev/null +++ b/.github/workflows/publish-mdq.yml @@ -0,0 +1,97 @@ +name: Publish mdq to npm + +on: + release: + types: [published] + +jobs: + publish: + if: startsWith(github.event.release.tag_name, 'mdq-') + runs-on: ubuntu-latest + + permissions: + contents: read + id-token: write + + steps: + - name: Checkout the released commit + uses: actions/checkout@v4 + with: + ref: ${{ github.event.release.tag_name }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 24 + registry-url: https://registry.npmjs.org + + - name: Upgrade npm for tokenless (OIDC) publishing + run: npm install -g npm@latest + + - name: Setup Bun + uses: oven-sh/setup-bun@v1 + with: + bun-version: latest + + - name: Install dependencies + run: bun install + + - name: Run mdq tests + run: bun test tests/unit/mdq/ + + - name: Set version from release + id: release + env: + TAG: ${{ github.event.release.tag_name }} + PRERELEASE: ${{ github.event.release.prerelease }} + run: | + VERSION="${TAG#mdq-}" + VERSION="${VERSION#v}" + echo "Publishing mdq $VERSION from release $TAG" + npm version "$VERSION" --no-git-tag-version + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + if [[ "$PRERELEASE" == "true" || "$VERSION" == *beta* || "$VERSION" == *pre* || "$VERSION" == *alpha* || "$VERSION" == *rc* ]]; then + echo "tag=beta" >> "$GITHUB_OUTPUT" + else + echo "tag=latest" >> "$GITHUB_OUTPUT" + fi + + - name: Stage mdq package + run: bun run build:mdq + + - name: Verify mdq tarball on Node.js + run: | + npm pack ./dist-mdq --pack-destination /tmp + mkdir -p /tmp/mdq-smoke && cd /tmp/mdq-smoke + npm init -y > /dev/null + npm install --ignore-scripts --no-audit --no-fund /tmp/mdq-*.tgz + printf '# Doc\n\n## API\n\n| Method | Path |\n|---|---|\n| GET | /users |\n' > sample.md + + echo "CLI, file argument" + ./node_modules/.bin/mdq 'h2' sample.md + + echo "CLI, stdin" + cat sample.md | ./node_modules/.bin/mdq 'section("API") table' --json + + echo "CLI, file argument while stdin is an open pipe (must not EAGAIN)" + ( sleep 3 ) | ./node_modules/.bin/mdq 'h2' sample.md + + echo "CLI, exit code 1 when nothing matches" + ./node_modules/.bin/mdq 'h5' sample.md && exit 1 || test $? -eq 1 + + echo "Library import" + node --input-type=module -e " + import { mdq } from 'mdq'; + const doc = mdq('---\nurl: /x\n---\n\n## A\n\n- one\n'); + if (doc.frontmatter().url !== '/x') throw new Error('frontmatter'); + if (doc.query('h2').count() !== 1) throw new Error('headings'); + if (!doc.list().addItem('two').toString().includes('- two')) throw new Error('addItem'); + " + + - name: Publish mdq to npm + run: | + if npm view "mdq@${{ steps.release.outputs.version }}" version > /dev/null 2>&1; then + echo "mdq@${{ steps.release.outputs.version }} is already published, skipping" + exit 0 + fi + npm publish ./dist-mdq --access public --tag ${{ steps.release.outputs.tag }} diff --git a/.gitignore b/.gitignore index 92f72598..3afdcf65 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ session.json # Build outputs dist/ dist-prima/ +dist-mdq/ build/ out/ .types-build/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 80592755..082d6bc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ ### New CLI Options - **`mdq`** — A new command for reading and editing markdown from the shell, the way `jq` reads JSON. + It ships as its own npm package, so it runs without installing anything: `npx mdq 'h2' README.md`. + It also works as a library — `import { mdq } from 'mdq'` — on Node 18 or newer. The first argument is a selector, the second an optional file (stdin is used when it is omitted). Matched markdown is printed by default. Exit codes compose like `grep`: `0` when something matched, `1` when nothing did, `2` for a bad selector or bad usage. @@ -53,6 +55,9 @@ - A search written as `/pattern/` is now case-sensitive unless it ends with `i`, matching how quoted searches already behaved. Previously every `/pattern/` ignored case whether it said so or not. - Searching a table now looks at its cells, not only its column titles. +- Reading a file no longer waits on standard input. Passing a filename while standard input was an + open pipe used to abort with an `EAGAIN` error on Node; input is now read only when no filename + is given. ## 2026-09-11 diff --git a/bin/mdq.ts b/bin/mdq.ts index e4309b01..d8e03608 100644 --- a/bin/mdq.ts +++ b/bin/mdq.ts @@ -1,11 +1,15 @@ -#!/usr/bin/env bun -import { readFileSync } from 'node:fs'; +#!/usr/bin/env node import { runMdq } from '../src/utils/mdq/cli.ts'; -let stdin = ''; -if (!process.stdin.isTTY) stdin = readFileSync(0, 'utf8'); +async function readStdin(): Promise { + if (process.stdin.isTTY) return ''; + process.stdin.setEncoding('utf8'); + let text = ''; + for await (const chunk of process.stdin) text += chunk; + return text; +} -const result = await runMdq(process.argv.slice(2), stdin); +const result = await runMdq(process.argv.slice(2), readStdin); if (result.output) { let text = result.output; if (!text.endsWith('\n')) text = `${text}\n`; diff --git a/package.json b/package.json index a1e7fac5..b75e5d9d 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,7 @@ }, "bin": { "explorbot": "./dist/bin/explorbot-cli.js", - "prima": "./dist/boat/prima/bin/prima-cli.js", - "mdq": "./dist/bin/mdq.js" + "prima": "./dist/boat/prima/bin/prima-cli.js" }, "files": [ "dist/", @@ -61,7 +60,8 @@ "check": "biome check .", "check:fix": "biome check --write .", "langfuse:export": "bun run .claude/skills/explorbot-debug/langfuse-export.ts", - "build:prima": "bun run scripts/build-prima-npm.ts" + "build:prima": "bun run scripts/build-prima-npm.ts", + "build:mdq": "bun run scripts/build-mdq-npm.ts" }, "keywords": ["cli", "react", "ink", "codeceptjs", "playwright"], "repository": { diff --git a/scripts/build-mdq-npm.ts b/scripts/build-mdq-npm.ts new file mode 100644 index 00000000..be4a70fe --- /dev/null +++ b/scripts/build-mdq-npm.ts @@ -0,0 +1,74 @@ +#!/usr/bin/env bun +import { cpSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; + +const ROOT = process.cwd(); +const STAGE = path.join(ROOT, 'dist-mdq'); +const PACKAGE = path.join(ROOT, 'src', 'utils', 'mdq'); +const EXTERNAL = ['marked', 'yaml', 'commander']; + +rmSync(STAGE, { recursive: true, force: true }); +mkdirSync(path.join(STAGE, 'bin'), { recursive: true }); + +async function bundle(entry: string, outfile: string) { + const result = await Bun.build({ + entrypoints: [entry], + target: 'node', + format: 'esm', + external: EXTERNAL, + outdir: path.dirname(outfile), + naming: path.basename(outfile), + }); + if (result.success) return; + for (const log of result.logs) console.error(log); + process.exit(1); +} + +await bundle(path.join(PACKAGE, 'query.ts'), path.join(STAGE, 'index.js')); +await bundle(path.join(ROOT, 'bin', 'mdq.ts'), path.join(STAGE, 'bin', 'mdq.js')); + +const cli = path.join(STAGE, 'bin', 'mdq.js'); +const shebanged = readFileSync(cli, 'utf8').replace(/^#!.*\n/, ''); +writeFileSync(cli, `#!/usr/bin/env node\n${shebanged}`, { mode: 0o755 }); + +const types = path.join(STAGE, 'types'); +const declarations = Bun.spawnSync([ + 'bunx', + 'tsc', + path.join(PACKAGE, 'query.ts'), + path.join(PACKAGE, 'edit.ts'), + '--declaration', + '--emitDeclarationOnly', + '--noCheck', + '--module', + 'esnext', + '--moduleResolution', + 'bundler', + '--target', + 'esnext', + '--allowImportingTsExtensions', + '--rewriteRelativeImportExtensions', + '--skipLibCheck', + '--outDir', + types, +]); +if (declarations.exitCode !== 0) { + console.error(declarations.stderr.toString()); + process.exit(1); +} + +for (const file of readdirSync(types)) { + const target = path.join(types, file); + writeFileSync(target, readFileSync(target, 'utf8').replaceAll("./edit.ts'", "./edit.js'")); +} + +const root = JSON.parse(readFileSync(path.join(ROOT, 'package.json'), 'utf8')); +const manifest = JSON.parse(readFileSync(path.join(PACKAGE, 'package.json'), 'utf8')); +manifest.version = root.version; +manifest.dependencies = {}; +for (const name of EXTERNAL) manifest.dependencies[name] = root.dependencies[name]; + +writeFileSync(path.join(STAGE, 'package.json'), `${JSON.stringify(manifest, null, 2)}\n`); +cpSync(path.join(PACKAGE, 'README.md'), path.join(STAGE, 'README.md')); + +console.log(`mdq ${manifest.version} staged in dist-mdq/`); diff --git a/src/utils/mdq/README.md b/src/utils/mdq/README.md index f27d9a3f..3bb9294d 100644 --- a/src/utils/mdq/README.md +++ b/src/utils/mdq/README.md @@ -2,6 +2,18 @@ Query and edit markdown with a selector language — jq, for markdown. +## Install + +```bash +npx mdq 'h2' README.md # no install +npm install mdq # as a library +npm install -g mdq # as a command +``` + +Node 18 or newer. Two dependencies: `marked` and `yaml`. + +## Use + ```js import { mdq } from 'mdq'; @@ -153,7 +165,3 @@ doc.setFrontmatter('wait', 2000); // comments and formatting survive because there is nothing to select. - **YAML frontmatter only.** TOML (`+++`) and JSON blocks are skipped from the token index but not parsed. - -## Dependencies - -`marked` and `yaml`. Nothing else. diff --git a/src/utils/mdq/cli.ts b/src/utils/mdq/cli.ts index 28140d91..e7fe53e8 100644 --- a/src/utils/mdq/cli.ts +++ b/src/utils/mdq/cli.ts @@ -4,7 +4,7 @@ import { MdqError, type MarkdownDoc, type Selection, mdq } from './query.ts'; const EDIT_FLAGS = ['remove', 'replace', 'insertBefore', 'insertAfter', 'prepend', 'append', 'addRow', 'addItem', 'set'] as const; -export async function runMdq(argv: string[], stdin: string): Promise { +export async function runMdq(argv: string[], readStdin: StdinReader): Promise { const program = new Command(); program .name('mdq') @@ -41,7 +41,8 @@ export async function runMdq(argv: string[], stdin: string): Promise selector = ''; } - let source = stdin; + let source = ''; + if (!file) source = await readStdin(); if (file) { try { source = readFileSync(file, 'utf8'); @@ -109,6 +110,8 @@ function finish(output: string, code: number, options: Record, file return { output: '', code }; } +export type StdinReader = () => Promise; + export interface CliResult { output: string; code: number; diff --git a/src/utils/mdq/package.json b/src/utils/mdq/package.json new file mode 100644 index 00000000..a048ac85 --- /dev/null +++ b/src/utils/mdq/package.json @@ -0,0 +1,28 @@ +{ + "name": "mdq", + "version": "0.0.0", + "description": "Query and edit markdown with a selector language - jq, for markdown", + "license": "MIT", + "type": "module", + "main": "./index.js", + "types": "./types/query.d.ts", + "exports": { + ".": { + "types": "./types/query.d.ts", + "import": "./index.js" + } + }, + "bin": { + "mdq": "./bin/mdq.js" + }, + "files": ["index.js", "types/", "bin/", "README.md"], + "engines": { + "node": ">=18" + }, + "keywords": ["markdown", "query", "selector", "cli", "jq", "frontmatter", "marked", "edit"], + "repository": { + "type": "git", + "url": "https://github.com/testomatio/explorbot", + "directory": "src/utils/mdq" + } +} diff --git a/tests/unit/mdq/cli.test.ts b/tests/unit/mdq/cli.test.ts index 262a6ee3..dbfc38a9 100644 --- a/tests/unit/mdq/cli.test.ts +++ b/tests/unit/mdq/cli.test.ts @@ -16,57 +16,57 @@ question? describe('reads', () => { it('prints matched markdown', async () => { - const result = await runMdq(['h2'], doc); + const result = await runMdq(['h2'], async () => doc); expect(result.output).toContain('## API'); expect(result.code).toBe(0); }); it('accepts a leading dot like jq', async () => { - expect((await runMdq(['.h2'], doc)).output).toContain('## API'); + expect((await runMdq(['.h2'], async () => doc)).output).toContain('## API'); }); it('prints rows as json', async () => { - const result = await runMdq(['section("API") table', '--json'], doc); + const result = await runMdq(['section("API") table', '--json'], async () => doc); expect(JSON.parse(result.output)).toEqual([{ Method: 'GET', Path: '/users' }]); }); it('prints a count', async () => { - expect((await runMdq(['h2', '--count'], doc)).output.trim()).toBe('2'); + expect((await runMdq(['h2', '--count'], async () => doc)).output.trim()).toBe('2'); }); it('prints unwrapped text', async () => { - expect((await runMdq(['h2', '--text'], doc)).output).not.toContain('##'); + expect((await runMdq(['h2', '--text'], async () => doc)).output).not.toContain('##'); }); it('prints frontmatter as json', async () => { - const result = await runMdq(['--frontmatter'], '---\nurl: /x\n---\n\n# T\n'); + const result = await runMdq(['--frontmatter'], async () => '---\nurl: /x\n---\n\n# T\n'); expect(JSON.parse(result.output)).toEqual({ url: '/x' }); }); }); describe('edits', () => { it('removes and prints the whole document', async () => { - const result = await runMdq(['section("FAQ")', '--remove'], doc); + const result = await runMdq(['section("FAQ")', '--remove'], async () => doc); expect(result.output).not.toContain('## FAQ'); expect(result.output).toContain('## API'); }); it('appends into a section', async () => { - expect((await runMdq(['section("FAQ")', '--append', 'answer!'], doc)).output).toContain('answer!'); + expect((await runMdq(['section("FAQ")', '--append', 'answer!'], async () => doc)).output).toContain('answer!'); }); it('adds a table row from json', async () => { - expect((await runMdq(['table', '--add-row', '{"Method":"POST","Path":"/s"}'], doc)).output).toContain('POST'); + expect((await runMdq(['table', '--add-row', '{"Method":"POST","Path":"/s"}'], async () => doc)).output).toContain('POST'); }); it('sets an entry', async () => { - const result = await runMdq(['blockquote', '--set', 'Container=.x'], '> Container: .old\n'); + const result = await runMdq(['blockquote', '--set', 'Container=.x'], async () => '> Container: .old\n'); expect(result.output).toContain('.x'); expect(result.output).not.toContain('.old'); }); it('refuses more than one edit at a time', async () => { - const result = await runMdq(['h2', '--remove', '--append', 'x'], doc); + const result = await runMdq(['h2', '--remove', '--append', 'x'], async () => doc); expect(result.code).toBe(2); expect(result.output).toContain('Only one edit'); }); @@ -74,26 +74,49 @@ describe('edits', () => { describe('exit codes', () => { it('returns 1 when nothing matches', async () => { - expect((await runMdq(['h5'], doc)).code).toBe(1); + expect((await runMdq(['h5'], async () => doc)).code).toBe(1); }); it('returns 2 on an unknown selector', async () => { - const result = await runMdq(['secton("A")'], doc); + const result = await runMdq(['secton("A")'], async () => doc); expect(result.code).toBe(2); expect(result.output).toContain('Unknown selector'); }); it('returns 1 when an edit matched nothing, leaving the document intact', async () => { - const result = await runMdq(['h5', '--remove'], doc); + const result = await runMdq(['h5', '--remove'], async () => doc); expect(result.code).toBe(1); expect(result.output).toBe(doc); }); it('returns 0 when an edit matched', async () => { - expect((await runMdq(['h2', '--remove'], doc)).code).toBe(0); + expect((await runMdq(['h2', '--remove'], async () => doc)).code).toBe(0); }); it('returns 2 without a selector', async () => { - expect((await runMdq([], doc)).code).toBe(2); + expect((await runMdq([], async () => doc)).code).toBe(2); + }); +}); + +describe('stdin', () => { + it('is not read when a file argument is given', async () => { + let read = false; + const reader = async () => { + read = true; + return ''; + }; + await runMdq(['h2', 'CLAUDE.md'], reader); + expect(read).toBe(false); + }); + + it('is read when no file argument is given', async () => { + let read = false; + const reader = async () => { + read = true; + return doc; + }; + const result = await runMdq(['h2'], reader); + expect(read).toBe(true); + expect(result.output).toContain('## API'); }); }); From 3238bada3e41419593bb3bc74fc183e4721d1b7d Mon Sep 17 00:00:00 2001 From: DavertMik Date: Thu, 17 Sep 2026 01:54:46 +0300 Subject: [PATCH 19/21] refactor(mdq): collapse back to one class query.ts was 790 lines: 20 module-level helper functions and three error classes sitting above a two-class split, with a 120-line hand-rolled character-scanning parser for the selector grammar. Now 577 lines with the class first and three module functions after it. - MarkdownDoc and Selection collapse back into one MarkdownQuery holding source plus matches, as it was before. The query engine (section scoping, item extraction, text matching, narrowing) moves inside as private methods, below the public API. - The selector parser drops from 120 lines of peek/advance/readUntilAny closures to one sticky regex and a 40-line loop. The grammar is closed, which CLAUDE.md says is a regex, not a parser. - blocks() is gone - it only existed to paper over the two-class split - and with it the toString/text asymmetry: toString() is always the document, text() the selection. - Document-level append/prepend are gone; they had no caller. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- scripts/build-mdq-npm.ts | 33 +- src/knowledge-tracker.ts | 2 +- src/utils/mdq/README.md | 35 +- src/utils/mdq/cli.ts | 8 +- src/utils/mdq/query.ts | 955 ++++++++++++++--------------------- tests/unit/mdq/edit.test.ts | 16 - tests/unit/mdq/sugar.test.ts | 17 +- 7 files changed, 409 insertions(+), 657 deletions(-) diff --git a/scripts/build-mdq-npm.ts b/scripts/build-mdq-npm.ts index be4a70fe..f4670579 100644 --- a/scripts/build-mdq-npm.ts +++ b/scripts/build-mdq-npm.ts @@ -1,4 +1,5 @@ #!/usr/bin/env bun +import { spawnSync } from 'node:child_process'; import { cpSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import path from 'node:path'; @@ -10,30 +11,28 @@ const EXTERNAL = ['marked', 'yaml', 'commander']; rmSync(STAGE, { recursive: true, force: true }); mkdirSync(path.join(STAGE, 'bin'), { recursive: true }); -async function bundle(entry: string, outfile: string) { - const result = await Bun.build({ - entrypoints: [entry], - target: 'node', - format: 'esm', - external: EXTERNAL, - outdir: path.dirname(outfile), - naming: path.basename(outfile), - }); - if (result.success) return; - for (const log of result.logs) console.error(log); +function run(command: string, args: string[]) { + const result = spawnSync(command, args, { encoding: 'utf8' }); + if (result.status === 0) return; + console.error(result.stderr || result.stdout); process.exit(1); } -await bundle(path.join(PACKAGE, 'query.ts'), path.join(STAGE, 'index.js')); -await bundle(path.join(ROOT, 'bin', 'mdq.ts'), path.join(STAGE, 'bin', 'mdq.js')); +function bundle(entry: string, outfile: string) { + const args = [entry, '--target', 'node', '--format', 'esm', '--outfile', outfile]; + for (const name of EXTERNAL) args.push('--external', name); + run('bun', ['build', ...args]); +} + +bundle(path.join(PACKAGE, 'query.ts'), path.join(STAGE, 'index.js')); +bundle(path.join(ROOT, 'bin', 'mdq.ts'), path.join(STAGE, 'bin', 'mdq.js')); const cli = path.join(STAGE, 'bin', 'mdq.js'); const shebanged = readFileSync(cli, 'utf8').replace(/^#!.*\n/, ''); writeFileSync(cli, `#!/usr/bin/env node\n${shebanged}`, { mode: 0o755 }); const types = path.join(STAGE, 'types'); -const declarations = Bun.spawnSync([ - 'bunx', +run('bunx', [ 'tsc', path.join(PACKAGE, 'query.ts'), path.join(PACKAGE, 'edit.ts'), @@ -52,10 +51,6 @@ const declarations = Bun.spawnSync([ '--outDir', types, ]); -if (declarations.exitCode !== 0) { - console.error(declarations.stderr.toString()); - process.exit(1); -} for (const file of readdirSync(types)) { const target = path.join(types, file); diff --git a/src/knowledge-tracker.ts b/src/knowledge-tracker.ts index bbe6009f..a1e5ada2 100644 --- a/src/knowledge-tracker.ts +++ b/src/knowledge-tracker.ts @@ -214,7 +214,7 @@ export class KnowledgeTracker { return this.knowledgeFiles.map((knowledge) => { const content = knowledge.content.trim(); - const firstLine = mdq(content).blocks().nodes()[0]?.text.split('\n')[0]?.trim() || ''; + const firstLine = mdq(content).nodes()[0]?.text.split('\n')[0]?.trim() || ''; return { url: knowledge.url || knowledge.endpoint || '', firstLine, diff --git a/src/utils/mdq/README.md b/src/utils/mdq/README.md index 3bb9294d..bff7722f 100644 --- a/src/utils/mdq/README.md +++ b/src/utils/mdq/README.md @@ -32,28 +32,20 @@ mdq 'section("API") table' --json README.md **Reads narrow, writes return the document.** -`mdq(source)` gives a `MarkdownDoc`. `query()` and the sugar methods narrow it to a -`Selection`. Every write returns a `MarkdownDoc` again, so edits chain and end with +`mdq(source)` gives a `MarkdownQuery` — one class, holding the document and the set of +blocks currently selected. `query()` and the sugar methods narrow that set. Every write +returns a fresh `MarkdownQuery` over the edited document, so edits chain and end with `toString()`: ```js mdq(source) - .query('section("API")') - .append('## Notes\n') - .query('blockquote[0]') - .remove() + .query('section("API")').append('## Notes\n') + .query('blockquote[0]').remove() .toString(); ``` -The two stringify differently, and the difference is deliberate: a `MarkdownDoc` gives the -whole document, a `Selection` gives only the markdown it matched. So a selection can be -passed straight back in as a fragment. - -```js -const fragment = mdq(other).query('section("Setup")'); -mdq(doc).query('h2').insertAfter(fragment); // inserts that section -String(mdq(doc).query('paragraph')); // the paragraphs, not the document -``` +`toString()` is always the whole document; `text()` is the markdown of the current +selection. ## Selectors @@ -121,12 +113,12 @@ over the node's text. Every sugar method takes one: `section` `heading` `paragra | `entries()` | `Key: value` lines of a block, keys lowercased | | `count()` / `exists()` | how many matched / whether any did | | `first()` / `last()` / `at(n)` / `slice(from, to)` | narrow the selection | -| `each()` | one single-match `Selection` per match | +| `each()` | one single-match query per match | | `preceding()` / `following()` | everything before the first / after the last match | ## Writing -Every one returns a `MarkdownDoc`. +Every one returns a `MarkdownQuery` over the edited document. | Method | Effect | | --- | --- | @@ -139,11 +131,10 @@ Every one returns a `MarkdownDoc`. | `addItem(text)` | append a list item, copying the existing marker | | `setEntry(key, value)` | set a `Key: value` line; `null` deletes it | -`MarkdownDoc` itself has `append` and `prepend`, which act on the whole document. - -Anything that takes markdown also takes a `MarkdownDoc`, so fragments compose without a -`toString()` hop. Writes never leave zero blank lines between blocks, and never more than -one — including inside fenced code blocks, which are left exactly as they are. +`prepend` and `append` need a section or list; on any other block they raise +`MdqOperationError`. Anything that takes markdown also takes a `MarkdownQuery`. Writes +never leave zero blank lines between blocks, and never more than one — including inside +fenced code blocks, which are left exactly as they are. ## Frontmatter diff --git a/src/utils/mdq/cli.ts b/src/utils/mdq/cli.ts index e7fe53e8..9917d583 100644 --- a/src/utils/mdq/cli.ts +++ b/src/utils/mdq/cli.ts @@ -1,6 +1,6 @@ import { readFileSync, writeFileSync } from 'node:fs'; import { Command } from 'commander'; -import { MdqError, type MarkdownDoc, type Selection, mdq } from './query.ts'; +import { type MarkdownQuery, MdqError, mdq } from './query.ts'; const EDIT_FLAGS = ['remove', 'replace', 'insertBefore', 'insertAfter', 'prepend', 'append', 'addRow', 'addItem', 'set'] as const; @@ -59,7 +59,7 @@ export async function runMdq(argv: string[], readStdin: StdinReader): Promise, file?: string): Promise { +async function apply(doc: MarkdownQuery, selector: string, options: Record, file?: string): Promise { if (options.frontmatter) return { output: `${JSON.stringify(doc.frontmatter(), null, 2)}\n`, code: 0 }; if (!selector) return { output: 'A selector is required', code: 2 }; @@ -73,7 +73,7 @@ async function apply(doc: MarkdownDoc, selector: string, options: Record): CliResult { +function read(selection: MarkdownQuery, options: Record): CliResult { let code = 1; if (selection.exists()) code = 0; if (options.count) return { output: `${selection.count()}\n`, code: 0 }; @@ -89,7 +89,7 @@ function read(selection: Selection, options: Record): CliResult { return { output: selection.text(), code }; } -function edit(selection: Selection, flag: string, options: Record): MarkdownDoc { +function edit(selection: MarkdownQuery, flag: string, options: Record): MarkdownQuery { if (flag === 'remove') return selection.remove(); if (flag === 'replace') return selection.replace(options.replace); if (flag === 'insertBefore') return selection.insertBefore(options.insertBefore); diff --git a/src/utils/mdq/query.ts b/src/utils/mdq/query.ts index e5be60ed..0f8f697d 100644 --- a/src/utils/mdq/query.ts +++ b/src/utils/mdq/query.ts @@ -1,508 +1,246 @@ import { type Token, type Tokens, marked } from 'marked'; -import { blockEnd, entryKey, insertAt, readFrontmatter, removeRanges, renderItem, renderTable, rewriteEntries, spliceRanges, splitFrontmatter, writeFrontmatter } from './edit.ts'; +import { blockEnd, insertAt, readFrontmatter, removeRanges, renderItem, renderTable, rewriteEntries, spliceRanges, splitFrontmatter, writeFrontmatter } from './edit.ts'; export { splitFrontmatter }; -export class MdqError extends Error {} - -export class MdqSelectorError extends MdqError { - index: number; - - constructor(message: string, index: number) { - super(message); - this.name = 'MdqSelectorError'; - this.index = index; - } -} - -export class MdqOperationError extends MdqError { - constructor(message: string) { - super(message); - this.name = 'MdqOperationError'; - } -} +const SELECTOR_NAMES = new Set(['section', 'heading', 'paragraph', 'table', 'list', 'item', 'code', 'blockquote', 'hr', 'html', 'comment']); -const SELECTORS = new Set(['section', 'heading', 'paragraph', 'table', 'list', 'item', 'code', 'blockquote', 'hr', 'html', 'comment']); +const TOKEN_TYPES: Record = { + heading: 'heading', + paragraph: 'paragraph', + table: 'table', + code: 'code', + list: 'list', + blockquote: 'blockquote', + hr: 'hr', + html: 'html', + item: 'list_item', +}; -function isKnownSelector(selector: string): boolean { - if (/^h[1-6]$/.test(selector)) return true; - if (/^section[1-6]?$/.test(selector)) return true; - return SELECTORS.has(selector); -} - -function isCommentToken(token: Token): boolean { - if (token.type !== 'html') return false; - return (((token as any).raw as string) || '').trimStart().startsWith('$/, '').trim(); -} - -export function parseQuery(input: string): QuerySegment[] { - const segments: QuerySegment[] = []; - let pos = 0; +const SEGMENT = /(\s*\.?)([A-Za-z]\w*)(?:\((!?)(~?)(?:"((?:[^"\\]|\\.)*)"|\/((?:[^/\\]|\\.)*)\/([a-z]*))\))?((?:\[[^\]]*\])*)\s*/y; - function peek(): string { - if (pos >= input.length) return ''; - return input[pos]; - } +export class MarkdownQuery { + private source: string; + private matches: MatchedRange[]; - function advance(): string { - return input[pos++] || ''; + constructor(source: string, matches?: MatchedRange[]) { + this.source = source; + this.matches = matches || buildTokenIndex(source); } - function skipWhitespace() { - while (pos < input.length && /\s/.test(input[pos])) pos++; + query(selector: string, matcher?: Matcher): MarkdownQuery { + const segments = parseQuery(selector); + if (matcher !== undefined && segments.length > 0) segments[segments.length - 1].textMatch = this.valueMatcher(matcher); + return new MarkdownQuery(this.source, this.run(this.expandSections(this.matches), segments)); } - function readIdentifier(): string { - const start = pos; - while (pos < input.length && /[a-zA-Z_\d]/.test(input[pos])) pos++; - return input.slice(start, pos); + section(matcher?: Matcher, options?: SelectorOptions): MarkdownQuery { + return this.query(`section${options?.depth || ''}`, matcher); } - function readQuotedString(): string { - const quote = advance(); - let result = ''; - while (pos < input.length && input[pos] !== quote) { - if (input[pos] === '\\') { - pos++; - result += input[pos] || ''; - } else { - result += input[pos]; - } - pos++; - } - if (pos < input.length) pos++; - return result; + heading(matcher?: Matcher, options?: SelectorOptions): MarkdownQuery { + if (options?.depth) return this.query(`h${options.depth}`, matcher); + return this.query('heading', matcher); } - function readUntilAny(chars: string): string { - const start = pos; - while (pos < input.length && !chars.includes(input[pos])) pos++; - return input.slice(start, pos); + paragraph(matcher?: Matcher): MarkdownQuery { + return this.query('paragraph', matcher); } - function parseTextMatcher(): TextMatcher { - let negated = false; - if (peek() === '!') { - negated = true; - advance(); - } - - if (peek() === '~') { - advance(); - const value = readQuotedString(); - return { mode: 'contains', value, negated }; - } - - if (peek() === '/') { - advance(); - let value = ''; - while (pos < input.length && input[pos] !== '/') { - value += input[pos]; - pos++; - } - if (pos < input.length) pos++; - const flagStart = pos; - while (pos < input.length && /[gimsuy]/.test(input[pos])) pos++; - return { mode: 'regex', value, negated, flags: input.slice(flagStart, pos) }; - } - - const value = readQuotedString(); - return { mode: 'exact', value, negated }; + table(matcher?: Matcher): MarkdownQuery { + return this.query('table', matcher); } - while (pos < input.length) { - skipWhitespace(); - if (pos >= input.length) break; - - if (peek() === '.') advance(); - const selectorStart = pos; - const selector = readIdentifier(); - if (!selector) throw new MdqSelectorError(`Unexpected character "${input[pos]}" in selector`, pos); - if (!isKnownSelector(selector)) throw new MdqSelectorError(`Unknown selector "${selector}"`, selectorStart); - - const segment: QuerySegment = { - selector: selector as SelectorType, - index: null, - slice: null, - }; - - if (peek() === '(') { - advance(); - segment.textMatch = parseTextMatcher(); - if (peek() === ')') advance(); - } - - while (peek() === '[') { - advance(); - const content = readUntilAny(']'); - if (/^-?\d*(:-?\d*)?$/.test(content) && content !== '') { - if (content.includes(':')) { - const colonIdx = content.indexOf(':'); - const fromStr = content.slice(0, colonIdx); - const toStr = content.slice(colonIdx + 1); - segment.slice = { - from: parseBound(fromStr), - to: parseBound(toStr), - }; - } else { - segment.index = Number.parseInt(content, 10); - } - } - if (peek() === ']') advance(); - } - - segments.push(segment); + list(matcher?: Matcher): MarkdownQuery { + return this.query('list', matcher); } - return segments; -} - -function parseBound(value: string): number | undefined { - if (!value) return undefined; - return Number.parseInt(value, 10); -} - -function toTextMatcher(matcher: Matcher): TextMatcher { - if (typeof matcher === 'function') return { mode: 'predicate', value: '', negated: false, predicate: matcher }; - if (matcher instanceof RegExp) return { mode: 'regex', value: matcher.source, negated: false, flags: matcher.flags }; - return { mode: 'exact', value: matcher, negated: false }; -} - -function applyMatcher(segments: QuerySegment[], matcher?: Matcher): QuerySegment[] { - if (matcher === undefined) return segments; - if (segments.length === 0) return segments; - segments[segments.length - 1].textMatch = toTextMatcher(matcher); - return segments; -} - -function matchText(text: string, matcher: TextMatcher): boolean { - let result: boolean; - - switch (matcher.mode) { - case 'exact': - result = text === matcher.value; - break; - case 'contains': - result = text.includes(matcher.value); - break; - case 'regex': - result = new RegExp(matcher.value, matcher.flags || '').test(text); - break; - case 'predicate': - result = matcher.predicate!(text); - break; - default: - result = false; - } - - if (matcher.negated) return !result; - return result; -} - -function getTokenText(token: Token): string { - const t = token as any; - switch (token.type) { - case 'heading': - case 'paragraph': - case 'code': - case 'blockquote': - case 'list_item': - return t.text || ''; - case 'html': - if (isCommentToken(token)) return commentBody(token); - return t.raw || ''; - case 'table': - return [...(t.header || []).map((h: any) => h.text), ...(t.rows || []).flatMap((row: any) => row.map((cell: any) => cell.text))].join(', '); - default: - return ''; + item(matcher?: Matcher): MarkdownQuery { + return this.query('item', matcher); } -} - -function getHeadingDepth(selector: string): number | null { - const match = selector.match(/^h([1-6])$/); - if (!match) return null; - return Number.parseInt(match[1], 10); -} -function isSectionSelector(selector: string): boolean { - return /^section[1-6]?$/.test(selector); -} - -function getSectionDepth(selector: string): number | null { - const match = selector.match(/^section([1-6])$/); - if (!match) return null; - return Number.parseInt(match[1], 10); -} - -function selectorToTokenType(selector: string): string | null { - if (/^h[1-6]$/.test(selector)) return 'heading'; - const map: Record = { - heading: 'heading', - paragraph: 'paragraph', - table: 'table', - code: 'code', - list: 'list', - blockquote: 'blockquote', - hr: 'hr', - html: 'html', - item: 'list_item', - }; - return map[selector] || null; -} - -export function buildTokenIndex(source: string): MatchedRange[] { - const { body, offset } = splitFrontmatter(source); - const tokens = marked.lexer(body); - const ranges: MatchedRange[] = []; - let cursor = offset; - - for (const token of tokens) { - const raw = (token as any).raw || ''; - if (token.type === 'space') { - const previous = ranges[ranges.length - 1]; - if (previous) previous.trailing = { start: cursor, length: raw.length }; - cursor += raw.length; - continue; - } - ranges.push({ token, start: cursor, length: raw.length }); - cursor += raw.length; + code(matcher?: Matcher): MarkdownQuery { + return this.query('code', matcher); } - return ranges; -} - -function computeSections(candidates: MatchedRange[], segment: QuerySegment): MatchedRange[] { - const sectionDepth = getSectionDepth(segment.selector as string); - const sections: MatchedRange[] = []; - - for (let i = 0; i < candidates.length; i++) { - const range = candidates[i]; - if (range.token.type !== 'heading') continue; - - const heading = range.token as Tokens.Heading; - if (sectionDepth !== null && heading.depth !== sectionDepth) continue; - if (segment.textMatch && !matchText(heading.text, segment.textMatch)) continue; - - const depth = heading.depth; - const innerTokens: MatchedRange[] = []; - let endOffset = range.start + range.length; - - for (let j = i + 1; j < candidates.length; j++) { - const nextRange = candidates[j]; - if (nextRange.token.type === 'heading' && (nextRange.token as Tokens.Heading).depth <= depth) break; - innerTokens.push(nextRange); - endOffset = nextRange.start + nextRange.length; - if (nextRange.trailing) endOffset = nextRange.trailing.start + nextRange.trailing.length; - } - - sections.push({ - token: range.token, - start: range.start, - length: endOffset - range.start, - innerTokens, - }); + blockquote(matcher?: Matcher): MarkdownQuery { + return this.query('blockquote', matcher); } - return sections; -} - -function extractListItems(candidates: MatchedRange[]): MatchedRange[] { - const items: MatchedRange[] = []; - - for (const range of candidates) { - if (range.token.type !== 'list') continue; - - const list = range.token as Tokens.List; - const listRaw = (range.token as any).raw as string; - let searchFrom = 0; - - for (const item of list.items) { - const itemRaw = (item as any).raw as string; - const idx = listRaw.indexOf(itemRaw, searchFrom); - if (idx === -1) continue; - - items.push({ - token: item as unknown as Token, - start: range.start + idx, - length: itemRaw.length, - }); - - searchFrom = idx + itemRaw.length; - } + comment(matcher?: Matcher): MarkdownQuery { + return this.query('comment', matcher); } - return items; -} - -function applyIndexSlice(matches: MatchedRange[], segment: QuerySegment): MatchedRange[] { - if (segment.index !== null) { - let idx = segment.index; - if (idx < 0) idx = matches.length + idx; - if (idx < 0 || idx >= matches.length) return []; - return [matches[idx]]; + html(matcher?: Matcher): MarkdownQuery { + return this.query('html', matcher); } - if (segment.slice) { - const { from, to } = segment.slice; - return matches.slice(from, to); + hr(): MarkdownQuery { + return this.query('hr'); } - return matches; -} - -function expandSectionRanges(matches: MatchedRange[]): MatchedRange[] { - let hasSection = false; - for (const m of matches) { - if (m.innerTokens) { - hasSection = true; - break; - } + text(): string { + return this.matches.map((range) => this.source.slice(range.start, range.start + range.length)).join(''); } - if (!hasSection) return matches; - const expanded: MatchedRange[] = []; - for (const m of matches) { - if (m.innerTokens) { - expanded.push({ token: m.token, start: m.start, length: ((m.token as any).raw || '').length }); - expanded.push(...m.innerTokens); - } else { - expanded.push(m); - } + nodes(): NodeInfo[] { + return this.matches.map((range) => { + const token = range.token as any; + if (token.type !== 'heading') return { type: token.type, depth: null, text: this.tokenText(range.token) }; + return { type: token.type, depth: token.depth, text: this.tokenText(range.token) }; + }); } - return expanded; -} - -function executeSegments(candidates: MatchedRange[], segments: QuerySegment[]): MatchedRange[] { - if (segments.length === 0) return candidates; - - const segment = segments[0]; - const remaining = segments.slice(1); - if (isSectionSelector(segment.selector as string)) { - const sections = computeSections(candidates, segment); - const indexed = applyIndexSlice(sections, segment); + rows(): Record[] { + const results: Record[] = []; - if (remaining.length === 0) return indexed; + for (const range of this.matches) { + if (range.token.type !== 'table') continue; - const results: MatchedRange[] = []; - for (const section of indexed) { - results.push(...executeSegments(section.innerTokens || [], remaining)); + const table = range.token as Tokens.Table; + const headers = table.header.map((cell) => cell.text); + for (const row of table.rows) { + const entry: Record = {}; + for (let i = 0; i < headers.length; i++) { + entry[headers[i]] = row[i]?.text ?? ''; + } + results.push(entry); + } } + return results; } - if (segment.selector === 'comment') { - let comments = candidates.filter((r) => isCommentToken(r.token)); - if (segment.textMatch) comments = comments.filter((r) => matchText(commentBody(r.token), segment.textMatch!)); - return executeSegments(applyIndexSlice(comments, segment), remaining); - } + entries(): Record { + const entries: Record = {}; - if (segment.selector === 'item') { - let items = extractListItems(candidates); - if (segment.textMatch) { - items = items.filter((r) => matchText(getTokenText(r.token), segment.textMatch!)); + for (const range of this.matches) { + for (const line of this.tokenText(range.token).split('\n')) { + const separator = line.indexOf(':'); + if (separator < 1) continue; + const value = line.slice(separator + 1).trim(); + if (value) entries[line.slice(0, separator).trim().toLowerCase()] = value; + } } - return executeSegments(applyIndexSlice(items, segment), remaining); - } - - const tokenType = selectorToTokenType(segment.selector as string); - if (!tokenType) return []; - - let matches = candidates.filter((r) => r.token.type === tokenType); - const depth = getHeadingDepth(segment.selector as string); - if (depth !== null) { - matches = matches.filter((r) => (r.token as any).depth === depth); + return entries; } - if (segment.textMatch) { - matches = matches.filter((r) => matchText(getTokenText(r.token), segment.textMatch!)); + frontmatter(): Record { + return readFrontmatter(this.source); } - return executeSegments(applyIndexSlice(matches, segment), remaining); -} + count(): number { + return this.matches.length; + } -export class MarkdownDoc { - protected source: string; + exists(): boolean { + return this.matches.length > 0; + } - constructor(source: string) { - this.source = source; + at(index: number): MarkdownQuery { + let resolved = index; + if (resolved < 0) resolved = this.matches.length + resolved; + if (resolved < 0 || resolved >= this.matches.length) return new MarkdownQuery(this.source, []); + return new MarkdownQuery(this.source, [this.matches[resolved]]); } - query(selector: string, matcher?: Matcher): Selection { - const segments = applyMatcher(parseQuery(selector), matcher); - const candidates = expandSectionRanges(buildTokenIndex(this.source)); - return new Selection(this.source, executeSegments(candidates, segments)); + slice(from?: number, to?: number): MarkdownQuery { + return new MarkdownQuery(this.source, this.matches.slice(from, to)); } - frontmatter(): Record { - return readFrontmatter(this.source); + first(): MarkdownQuery { + return this.slice(0, 1); } - setFrontmatter(key: string, value: unknown): MarkdownDoc { - return new MarkdownDoc(writeFrontmatter(this.source, key, value)); + last(): MarkdownQuery { + return this.slice(-1); } - blocks(): Selection { - return new Selection(this.source); + each(): MarkdownQuery[] { + return this.matches.map((range) => new MarkdownQuery(this.source, [range])); } - section(matcher?: Matcher, options?: SelectorOptions): Selection { - return this.query(`section${options?.depth || ''}`, matcher); + preceding(): MarkdownQuery { + if (this.matches.length === 0) return new MarkdownQuery(this.source, []); + const cutoff = this.matches[0].start; + return new MarkdownQuery( + this.source, + buildTokenIndex(this.source).filter((range) => range.start + range.length <= cutoff) + ); } - heading(matcher?: Matcher, options?: SelectorOptions): Selection { - if (options?.depth) return this.query(`h${options.depth}`, matcher); - return this.query('heading', matcher); + following(): MarkdownQuery { + if (this.matches.length === 0) return new MarkdownQuery(this.source, []); + const last = this.matches[this.matches.length - 1]; + const cutoff = last.start + last.length; + return new MarkdownQuery( + this.source, + buildTokenIndex(this.source).filter((range) => range.start >= cutoff) + ); } - paragraph(matcher?: Matcher): Selection { - return this.query('paragraph', matcher); + replace(content: Markdown): MarkdownQuery { + return this.replaceEach(() => content); } - table(matcher?: Matcher): Selection { - return this.query('table', matcher); + replaceEach(replacer: (match: MarkdownQuery, index: number) => Markdown): MarkdownQuery { + return new MarkdownQuery(spliceRanges(this.source, this.matches, (range, index) => String(replacer(new MarkdownQuery(this.source, [range]), index)))); } - list(matcher?: Matcher): Selection { - return this.query('list', matcher); + remove(): MarkdownQuery { + return new MarkdownQuery(removeRanges(this.source, this.matches)); } - item(matcher?: Matcher): Selection { - return this.query('item', matcher); + insertBefore(markdown: Markdown): MarkdownQuery { + return this.insertAll((range) => range.start, markdown); } - code(matcher?: Matcher): Selection { - return this.query('code', matcher); + insertAfter(markdown: Markdown): MarkdownQuery { + return this.insertAll((range) => blockEnd(range), markdown); } - blockquote(matcher?: Matcher): Selection { - return this.query('blockquote', matcher); + prepend(markdown: Markdown): MarkdownQuery { + return this.insertAll((range) => this.containerStart(range), markdown); } - comment(matcher?: Matcher): Selection { - return this.query('comment', matcher); + append(markdown: Markdown): MarkdownQuery { + return this.insertAll((range) => this.containerEnd(range), markdown); } - html(matcher?: Matcher): Selection { - return this.query('html', matcher); + addRow(row: Record): MarkdownQuery { + return new MarkdownQuery( + spliceRanges(this.source, this.matches, (range) => { + if (range.token.type !== 'table') throw new MdqOperationError(`addRow needs a table, got ${range.token.type}`); + const table = range.token as Tokens.Table; + const headers = table.header.map((cell) => cell.text); + const existing = table.rows.map((cells) => headers.map((_, index) => cells[index]?.text || '')); + return renderTable(headers, [...existing, headers.map((header) => row[header] || '')], table.align); + }) + ); } - hr(): Selection { - return this.query('hr'); + addItem(text: string): MarkdownQuery { + return new MarkdownQuery( + spliceRanges(this.source, this.matches, (range) => { + if (range.token.type !== 'list') throw new MdqOperationError(`addItem needs a list, got ${range.token.type}`); + const raw = (((range.token as any).raw as string) || '').trimEnd(); + return `${raw}\n${renderItem(raw, text)}\n`; + }) + ); } - append(markdown: Markdown): MarkdownDoc { - return new MarkdownDoc(insertAt(this.source, this.source.length, String(markdown))); + setEntry(key: string, value: string | null): MarkdownQuery { + return this.replaceEach((match) => { + const token = match.matches[0].token; + return rewriteEntries(this.tokenText(token), key, value, token.type === 'blockquote'); + }); } - prepend(markdown: Markdown): MarkdownDoc { - return new MarkdownDoc(insertAt(this.source, splitFrontmatter(this.source).offset, String(markdown))); + setFrontmatter(key: string, value: unknown): MarkdownQuery { + return new MarkdownQuery(writeFrontmatter(this.source, key, value)); } toString(): string { @@ -512,53 +250,15 @@ export class MarkdownDoc { valueOf(): string { return this.source; } -} - -export class Selection extends MarkdownDoc { - private matches: MatchedRange[]; - - constructor(source: string, matches?: MatchedRange[]) { - super(source); - this.matches = matches || buildTokenIndex(source); - } - - query(selector: string, matcher?: Matcher): Selection { - const segments = applyMatcher(parseQuery(selector), matcher); - const candidates = expandSectionRanges(this.matches); - return new Selection(this.source, executeSegments(candidates, segments)); - } - - text(): string { - return this.matches.map((r) => this.source.slice(r.start, r.start + r.length)).join(''); - } - - toString(): string { - return this.text(); - } /** @deprecated Use text(). */ get(): string { return this.text(); } - rows(): Record[] { - const results: Record[] = []; - - for (const range of this.matches) { - if (range.token.type !== 'table') continue; - - const table = range.token as Tokens.Table; - const headers = table.header.map((h) => h.text); - for (const row of table.rows) { - const obj: Record = {}; - for (let i = 0; i < headers.length; i++) { - obj[headers[i]] = row[i]?.text ?? ''; - } - results.push(obj); - } - } - - return results; + /** @deprecated Use nodes(). */ + meta(): NodeInfo[] { + return this.nodes(); } /** @deprecated Use rows(). */ @@ -566,168 +266,178 @@ export class Selection extends MarkdownDoc { return this.rows(); } - entries(): Record { - const entries: Record = {}; - - for (const range of this.matches) { - for (const line of getTokenText(range.token).split('\n')) { - const key = entryKey(line); - if (!key) continue; - const value = line.slice(line.indexOf(':') + 1).trim(); - if (value) entries[key] = value; - } - } - - return entries; - } - /** @deprecated Use entries(). */ keyValue(): Record { return this.entries(); } - setEntry(key: string, value: string | null): MarkdownDoc { - return this.replaceEach((match) => { - const token = match.matches[0].token; - return rewriteEntries(getTokenText(token), key, value, token.type === 'blockquote'); - }); - } - /** @deprecated Use setEntry(). */ - setKeyValue(key: string, value: string | null): MarkdownDoc { + setKeyValue(key: string, value: string | null): MarkdownQuery { return this.setEntry(key, value); } - remove(): MarkdownDoc { - return new MarkdownDoc(removeRanges(this.source, this.matches)); + /** @deprecated Use preceding(). */ + before(): MarkdownQuery { + return this.preceding(); } - insertBefore(markdown: Markdown): MarkdownDoc { - return this.insertEach((range) => range.start, markdown); + /** @deprecated Use following(). */ + after(): MarkdownQuery { + return this.following(); } - insertAfter(markdown: Markdown): MarkdownDoc { - return this.insertEach((range) => blockEnd(range), markdown); - } + private run(candidates: MatchedRange[], segments: QuerySegment[]): MatchedRange[] { + if (segments.length === 0) return candidates; - prepend(markdown: Markdown): MarkdownDoc { - return this.insertEach((range) => this.containerStart(range), markdown); - } + const segment = segments[0]; + const remaining = segments.slice(1); - append(markdown: Markdown): MarkdownDoc { - return this.insertEach((range) => this.containerEnd(range), markdown); - } + if (/^section[1-6]?$/.test(segment.selector)) { + const sections = this.narrow(this.sectionsOf(candidates, segment), segment); + if (remaining.length === 0) return sections; - addRow(row: Record): MarkdownDoc { - return new MarkdownDoc( - spliceRanges(this.source, this.matches, (range) => { - if (range.token.type !== 'table') throw new MdqOperationError(`addRow needs a table, got ${range.token.type}`); - const table = range.token as Tokens.Table; - const headers = table.header.map((cell) => cell.text); - const existing = table.rows.map((cells) => headers.map((_, index) => cells[index]?.text || '')); - return renderTable(headers, [...existing, headers.map((header) => row[header] || '')], table.align); - }) - ); - } + const results: MatchedRange[] = []; + for (const section of sections) results.push(...this.run(section.innerTokens || [], remaining)); + return results; + } - addItem(text: string): MarkdownDoc { - return new MarkdownDoc( - spliceRanges(this.source, this.matches, (range) => { - if (range.token.type !== 'list') throw new MdqOperationError(`addItem needs a list, got ${range.token.type}`); - const raw = (((range.token as any).raw as string) || '').replace(/\s+$/, ''); - return `${raw}\n${renderItem(raw, text)}\n`; - }) - ); - } + if (segment.selector === 'comment') { + const comments = candidates.filter((range) => this.isComment(range.token)); + return this.run(this.narrow(this.filterByText(comments, segment), segment), remaining); + } - replace(content: Markdown): MarkdownDoc { - return this.replaceEach(() => content); - } + if (segment.selector === 'item') { + return this.run(this.narrow(this.filterByText(this.itemsOf(candidates), segment), segment), remaining); + } - replaceEach(replacer: (match: Selection, index: number) => Markdown): MarkdownDoc { - return new MarkdownDoc(spliceRanges(this.source, this.matches, (range, index) => String(replacer(new Selection(this.source, [range]), index)))); - } + const depth = segment.selector.match(/^h([1-6])$/); + let type = TOKEN_TYPES[segment.selector]; + if (depth) type = 'heading'; + if (!type) return []; - count(): number { - return this.matches.length; - } + let matches = candidates.filter((range) => range.token.type === type); + if (depth) matches = matches.filter((range) => (range.token as any).depth === Number.parseInt(depth[1], 10)); - exists(): boolean { - return this.matches.length > 0; + return this.run(this.narrow(this.filterByText(matches, segment), segment), remaining); } - at(index: number): Selection { - let resolved = index; - if (resolved < 0) resolved = this.matches.length + resolved; - if (resolved < 0 || resolved >= this.matches.length) return new Selection(this.source, []); - return new Selection(this.source, [this.matches[resolved]]); - } + private sectionsOf(candidates: MatchedRange[], segment: QuerySegment): MatchedRange[] { + const wanted = segment.selector.match(/^section([1-6])$/); + const sections: MatchedRange[] = []; - slice(from?: number, to?: number): Selection { - return new Selection(this.source, this.matches.slice(from, to)); - } + for (let i = 0; i < candidates.length; i++) { + const range = candidates[i]; + if (range.token.type !== 'heading') continue; + + const heading = range.token as Tokens.Heading; + if (wanted && heading.depth !== Number.parseInt(wanted[1], 10)) continue; + if (segment.textMatch && !this.matchText(heading.text, segment.textMatch)) continue; - first(): Selection { - return new Selection(this.source, this.matches.slice(0, 1)); + const innerTokens: MatchedRange[] = []; + let end = range.start + range.length; + + for (let j = i + 1; j < candidates.length; j++) { + const next = candidates[j]; + if (next.token.type === 'heading' && (next.token as Tokens.Heading).depth <= heading.depth) break; + innerTokens.push(next); + end = blockEnd(next); + } + + sections.push({ token: range.token, start: range.start, length: end - range.start, innerTokens }); + } + + return sections; } - last(): Selection { - return new Selection(this.source, this.matches.slice(-1)); + private itemsOf(candidates: MatchedRange[]): MatchedRange[] { + const items: MatchedRange[] = []; + + for (const range of candidates) { + if (range.token.type !== 'list') continue; + + const raw = (range.token as any).raw as string; + let cursor = 0; + + for (const item of (range.token as Tokens.List).items) { + const itemRaw = (item as any).raw as string; + const at = raw.indexOf(itemRaw, cursor); + if (at === -1) continue; + items.push({ token: item as unknown as Token, start: range.start + at, length: itemRaw.length }); + cursor = at + itemRaw.length; + } + } + + return items; } - preceding(): Selection { - if (this.matches.length === 0) return new Selection(this.source, []); - const cutoff = this.matches[0].start; - return new Selection( - this.source, - buildTokenIndex(this.source).filter((r) => r.start + r.length <= cutoff) - ); + private expandSections(matches: MatchedRange[]): MatchedRange[] { + if (!matches.some((range) => range.innerTokens)) return matches; + + const expanded: MatchedRange[] = []; + for (const range of matches) { + if (!range.innerTokens) { + expanded.push(range); + continue; + } + expanded.push({ token: range.token, start: range.start, length: ((range.token as any).raw || '').length }); + expanded.push(...range.innerTokens); + } + return expanded; } - /** @deprecated Use preceding(). */ - before(): Selection { - return this.preceding(); + private narrow(matches: MatchedRange[], segment: QuerySegment): MatchedRange[] { + if (segment.index !== null) { + let index = segment.index; + if (index < 0) index = matches.length + index; + if (index < 0 || index >= matches.length) return []; + return [matches[index]]; + } + if (segment.slice) return matches.slice(segment.slice.from, segment.slice.to); + return matches; } - following(): Selection { - if (this.matches.length === 0) return new Selection(this.source, []); - const lastMatch = this.matches[this.matches.length - 1]; - const cutoff = lastMatch.start + lastMatch.length; - return new Selection( - this.source, - buildTokenIndex(this.source).filter((r) => r.start >= cutoff) - ); + private filterByText(matches: MatchedRange[], segment: QuerySegment): MatchedRange[] { + if (!segment.textMatch) return matches; + return matches.filter((range) => this.matchText(this.tokenText(range.token), segment.textMatch!)); } - /** @deprecated Use following(). */ - after(): Selection { - return this.following(); + private matchText(text: string, matcher: TextMatcher): boolean { + let result = false; + if (matcher.mode === 'exact') result = text === matcher.value; + if (matcher.mode === 'contains') result = text.includes(matcher.value); + if (matcher.mode === 'regex') result = new RegExp(matcher.value, matcher.flags || '').test(text); + if (matcher.mode === 'predicate') result = matcher.predicate!(text); + if (matcher.negated) return !result; + return result; } - each(): Selection[] { - return this.matches.map((m) => new Selection(this.source, [m])); + private valueMatcher(matcher: Matcher): TextMatcher { + if (typeof matcher === 'function') return { mode: 'predicate', value: '', negated: false, predicate: matcher }; + if (matcher instanceof RegExp) return { mode: 'regex', value: matcher.source, negated: false, flags: matcher.flags }; + return { mode: 'exact', value: matcher, negated: false }; } - nodes(): NodeInfo[] { - return this.matches.map((range) => { - const token = range.token as any; - if (token.type !== 'heading') return { type: token.type, depth: null, text: getTokenText(range.token) }; - return { type: token.type, depth: token.depth, text: getTokenText(range.token) }; - }); + private tokenText(token: Token): string { + const value = token as any; + if (this.isComment(token)) return (value.raw as string).trim().replace(/^$/, '').trim(); + if (token.type === 'html') return value.raw || ''; + if (token.type === 'table') return [...(value.header || []).map((cell: any) => cell.text), ...(value.rows || []).flatMap((row: any) => row.map((cell: any) => cell.text))].join(', '); + if (TEXT_TOKENS.has(token.type)) return value.text || ''; + return ''; } - /** @deprecated Use nodes(). */ - meta(): NodeInfo[] { - return this.nodes(); + private isComment(token: Token): boolean { + if (token.type !== 'html') return false; + return (((token as any).raw as string) || '').trimStart().startsWith(' more`) is From 4bf74d7cfbf25a07e2d176e0abacc5e1477ec3a1 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 18 Sep 2026 01:05:56 +0300 Subject: [PATCH 21/21] refactor(mdq): one editor class, consolidated constants edit.ts becomes MarkdownEditor: an object holding the source with remove/replace/insert/frontmatter/setFrontmatter on it, and the pure helpers (blockEnd, splitFrontmatter, table, item, entries) as statics. query.ts delegates every write to it instead of importing ten loose functions. Constants were largely restating each other. TOKEN_TYPES was eight identity pairs plus one real mapping, so only the mapping survives as TOKEN_ALIASES. TEXT_TOKENS is gone - tokenText falls through to the token's own text field, which is what the whitelist was approximating. The three that remain sit at the end of the file beside the types. Seven repeated @deprecated lines become one comment over the alias group. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FhJNfxLquFknSLJ6V8J9iD --- src/utils/mdq/edit.ts | 238 ++++++++++++++++------------- src/utils/mdq/query.ts | 79 ++++------ tests/unit/mdq/frontmatter.test.ts | 8 +- 3 files changed, 162 insertions(+), 163 deletions(-) diff --git a/src/utils/mdq/edit.ts b/src/utils/mdq/edit.ts index 41db15b1..152709a4 100644 --- a/src/utils/mdq/edit.ts +++ b/src/utils/mdq/edit.ts @@ -1,134 +1,154 @@ import YAML from 'yaml'; import type { MatchedRange } from './query.ts'; -export function splitFrontmatter(source: string): FrontmatterSplit { - if (!source.startsWith('---')) return { raw: '', body: source, offset: 0 }; - const match = source.match(/^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/); - if (!match) return { raw: '', body: source, offset: 0 }; - return { raw: match[1], body: source.slice(match[0].length), offset: match[0].length }; -} +export class MarkdownEditor { + private source: string; -function dedupeRanges(ranges: MatchedRange[]): MatchedRange[] { - const sorted = [...ranges].sort((a, b) => a.start - b.start); - const kept: MatchedRange[] = []; - let lastEnd = -1; - for (const range of sorted) { - if (range.start < lastEnd) continue; - kept.push(range); - lastEnd = range.start + range.length; + constructor(source: string) { + this.source = source; } - return kept; -} -export function blockEnd(range: MatchedRange): number { - if (range.trailing) return range.trailing.start + range.trailing.length; - return range.start + range.length; -} + static splitFrontmatter(source: string): FrontmatterSplit { + if (!source.startsWith('---')) return { raw: '', body: source, offset: 0 }; + const match = source.match(/^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/); + if (!match) return { raw: '', body: source, offset: 0 }; + return { raw: match[1], body: source.slice(match[0].length), offset: match[0].length }; + } -export function normalizeBlock(markdown: string): string { - return `${markdown.trimEnd()}\n`; -} + static blockEnd(range: MatchedRange): number { + if (range.trailing) return range.trailing.start + range.trailing.length; + return range.start + range.length; + } -export function removeRanges(source: string, ranges: MatchedRange[]): string { - const ordered = dedupeRanges(ranges); - let result = source; - for (let i = ordered.length - 1; i >= 0; i--) { - const range = ordered[i]; - const head = result.slice(0, range.start); - const tail = result.slice(blockEnd(range)); - if (tail) { - result = head + tail; - continue; - } - if (!head) { - result = ''; - continue; + static table(headers: string[], rows: string[][], align: (string | null)[]): string { + const widths = headers.map((header, index) => Math.max(header.length, 3, ...rows.map((row) => (row[index] || '').length))); + const line = (cells: string[]) => `| ${cells.map((cell, index) => (cell || '').padEnd(widths[index])).join(' | ')} |`; + const divider = `| ${widths.map((width, index) => MarkdownEditor.dashes(align[index], width)).join(' | ')} |`; + return `${[line(headers), divider, ...rows.map(line)].join('\n')}\n`; + } + + static item(listRaw: string, text: string): string { + const lines = listRaw.split('\n').filter((line) => line.trim()); + const last = lines[lines.length - 1] || '- x'; + const ordered = last.match(/^(\s*)(\d+)([.)])\s/); + if (ordered) return `${ordered[1]}${Number.parseInt(ordered[2], 10) + 1}${ordered[3]} ${text}`; + const bullet = last.match(/^(\s*)([-*+])\s/); + if (!bullet) return `- ${text}`; + return `${bullet[1]}${bullet[2]} ${text}`; + } + + static entries(text: string, key: string, value: string | null, isBlockquote: boolean): string { + const lines = text + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); + + const index = lines.findIndex((line) => MarkdownEditor.entryKey(line) === key.toLowerCase()); + if (index < 0 && value) lines.push(`${key}: ${value}`); + if (index >= 0 && value) lines[index] = `${key}: ${value}`; + if (index >= 0 && !value) lines.splice(index, 1); + + if (!isBlockquote) return lines.join('\n'); + return lines.map((line) => `> ${line}`).join('\n'); + } + + remove(ranges: MatchedRange[]): string { + const ordered = this.dedupe(ranges); + let result = this.source; + + for (let i = ordered.length - 1; i >= 0; i--) { + const range = ordered[i]; + const head = result.slice(0, range.start); + const tail = result.slice(MarkdownEditor.blockEnd(range)); + if (tail) { + result = head + tail; + continue; + } + if (!head) { + result = ''; + continue; + } + result = `${head.trimEnd()}\n`; } - result = `${head.trimEnd()}\n`; + + return result; } - return result; -} -export function insertAt(source: string, offset: number, markdown: string): string { - const block = normalizeBlock(markdown); - const before = source.slice(0, offset).replace(/\n+$/, ''); - const after = source.slice(offset).replace(/^\n+/, ''); - if (!before) return `${block}\n${after}`; - if (!after) return `${before}\n\n${block}`; - return `${before}\n\n${block}\n${after}`; -} + replace(ranges: MatchedRange[], render: (range: MatchedRange, index: number) => string): string { + const ordered = this.dedupe(ranges); + const rendered = ordered.map(render); + let result = this.source; -export function spliceRanges(source: string, ranges: MatchedRange[], render: (range: MatchedRange, index: number) => string): string { - const ordered = dedupeRanges(ranges); - const rendered = ordered.map(render); - let result = source; - for (let i = ordered.length - 1; i >= 0; i--) { - const range = ordered[i]; - result = result.slice(0, range.start) + rendered[i] + result.slice(range.start + range.length); + for (let i = ordered.length - 1; i >= 0; i--) { + const range = ordered[i]; + result = result.slice(0, range.start) + rendered[i] + result.slice(range.start + range.length); + } + + return result; } - return result; -} -export function renderTable(headers: string[], rows: string[][], align: (string | null)[]): string { - const widths = headers.map((header, index) => Math.max(header.length, 3, ...rows.map((row) => (row[index] || '').length))); - const line = (cells: string[]) => `| ${cells.map((cell, index) => (cell || '').padEnd(widths[index])).join(' | ')} |`; - const divider = `| ${widths.map((width, index) => dashes(align[index], width)).join(' | ')} |`; - return `${[line(headers), divider, ...rows.map(line)].join('\n')}\n`; -} + insert(offsets: number[], markdown: string): string { + const ordered = [...offsets].sort((a, b) => a - b); + let result = this.source; + for (let i = ordered.length - 1; i >= 0; i--) { + result = MarkdownEditor.spliceIn(result, ordered[i], markdown); + } + return result; + } -export function renderItem(listRaw: string, text: string): string { - const lines = listRaw.split('\n').filter((line) => line.trim()); - const last = lines[lines.length - 1] || '- x'; - const ordered = last.match(/^(\s*)(\d+)([.)])\s/); - if (ordered) return `${ordered[1]}${Number.parseInt(ordered[2], 10) + 1}${ordered[3]} ${text}`; - const bullet = last.match(/^(\s*)([-*+])\s/); - if (!bullet) return `- ${text}`; - return `${bullet[1]}${bullet[2]} ${text}`; -} + frontmatter(): Record { + const { raw } = MarkdownEditor.splitFrontmatter(this.source); + if (!raw) return {}; + return (YAML.parseDocument(raw).toJS() as Record) || {}; + } -function dashes(alignment: string | null, width: number): string { - if (alignment === 'center') return `:${'-'.repeat(Math.max(width - 2, 1))}:`; - if (alignment === 'left') return `:${'-'.repeat(Math.max(width - 1, 1))}`; - if (alignment === 'right') return `${'-'.repeat(Math.max(width - 1, 1))}:`; - return '-'.repeat(width); -} + setFrontmatter(key: string, value: unknown): string { + const { raw, body, offset } = MarkdownEditor.splitFrontmatter(this.source); + let document = new YAML.Document({}); + if (raw) document = YAML.parseDocument(raw); + if (value === null) document.delete(key); + if (value !== null) document.set(key, value); -export function entryKey(line: string): string | null { - const separator = line.indexOf(':'); - if (separator < 1) return null; - return line.slice(0, separator).trim().toLowerCase(); -} + const rendered = document.toString().trimEnd(); + if (!offset) return `---\n${rendered}\n---\n\n${this.source}`; + return `---\n${rendered}\n---\n${body}`; + } -export function rewriteEntries(text: string, key: string, value: string | null, isBlockquote: boolean): string { - const lines = text - .split('\n') - .map((line) => line.trim()) - .filter(Boolean); + private static spliceIn(source: string, offset: number, markdown: string): string { + const block = `${markdown.trimEnd()}\n`; + const before = source.slice(0, offset).replace(/\n+$/, ''); + const after = source.slice(offset).replace(/^\n+/, ''); + if (!before) return `${block}\n${after}`; + if (!after) return `${before}\n\n${block}`; + return `${before}\n\n${block}\n${after}`; + } - const index = lines.findIndex((line) => entryKey(line) === key.toLowerCase()); - if (index < 0 && value) lines.push(`${key}: ${value}`); - if (index >= 0 && value) lines[index] = `${key}: ${value}`; - if (index >= 0 && !value) lines.splice(index, 1); + private static dashes(alignment: string | null, width: number): string { + if (alignment === 'center') return `:${'-'.repeat(Math.max(width - 2, 1))}:`; + if (alignment === 'left') return `:${'-'.repeat(Math.max(width - 1, 1))}`; + if (alignment === 'right') return `${'-'.repeat(Math.max(width - 1, 1))}:`; + return '-'.repeat(width); + } - if (!isBlockquote) return lines.join('\n'); - return lines.map((line) => `> ${line}`).join('\n'); -} + private static entryKey(line: string): string | null { + const separator = line.indexOf(':'); + if (separator < 1) return null; + return line.slice(0, separator).trim().toLowerCase(); + } -export function readFrontmatter(source: string): Record { - const { raw } = splitFrontmatter(source); - if (!raw) return {}; - return (YAML.parseDocument(raw).toJS() as Record) || {}; -} + private dedupe(ranges: MatchedRange[]): MatchedRange[] { + const sorted = [...ranges].sort((a, b) => a.start - b.start); + const kept: MatchedRange[] = []; + let lastEnd = -1; + + for (const range of sorted) { + if (range.start < lastEnd) continue; + kept.push(range); + lastEnd = range.start + range.length; + } -export function writeFrontmatter(source: string, key: string, value: unknown): string { - const { raw, body, offset } = splitFrontmatter(source); - let document = new YAML.Document({}); - if (raw) document = YAML.parseDocument(raw); - if (value === null) document.delete(key); - if (value !== null) document.set(key, value); - const rendered = document.toString().trimEnd(); - if (!offset) return `---\n${rendered}\n---\n\n${source}`; - return `---\n${rendered}\n---\n${body}`; + return kept; + } } export interface FrontmatterSplit { diff --git a/src/utils/mdq/query.ts b/src/utils/mdq/query.ts index 0f8f697d..823f0380 100644 --- a/src/utils/mdq/query.ts +++ b/src/utils/mdq/query.ts @@ -1,25 +1,7 @@ import { type Token, type Tokens, marked } from 'marked'; -import { blockEnd, insertAt, readFrontmatter, removeRanges, renderItem, renderTable, rewriteEntries, spliceRanges, splitFrontmatter, writeFrontmatter } from './edit.ts'; +import { MarkdownEditor } from './edit.ts'; -export { splitFrontmatter }; - -const SELECTOR_NAMES = new Set(['section', 'heading', 'paragraph', 'table', 'list', 'item', 'code', 'blockquote', 'hr', 'html', 'comment']); - -const TOKEN_TYPES: Record = { - heading: 'heading', - paragraph: 'paragraph', - table: 'table', - code: 'code', - list: 'list', - blockquote: 'blockquote', - hr: 'hr', - html: 'html', - item: 'list_item', -}; - -const TEXT_TOKENS = new Set(['heading', 'paragraph', 'code', 'blockquote', 'list_item']); - -const SEGMENT = /(\s*\.?)([A-Za-z]\w*)(?:\((!?)(~?)(?:"((?:[^"\\]|\\.)*)"|\/((?:[^/\\]|\\.)*)\/([a-z]*))\))?((?:\[[^\]]*\])*)\s*/y; +export { MarkdownEditor }; export class MarkdownQuery { private source: string; @@ -129,7 +111,7 @@ export class MarkdownQuery { } frontmatter(): Record { - return readFrontmatter(this.source); + return this.editor().frontmatter(); } count(): number { @@ -187,11 +169,11 @@ export class MarkdownQuery { } replaceEach(replacer: (match: MarkdownQuery, index: number) => Markdown): MarkdownQuery { - return new MarkdownQuery(spliceRanges(this.source, this.matches, (range, index) => String(replacer(new MarkdownQuery(this.source, [range]), index)))); + return new MarkdownQuery(this.editor().replace(this.matches, (range, index) => String(replacer(new MarkdownQuery(this.source, [range]), index)))); } remove(): MarkdownQuery { - return new MarkdownQuery(removeRanges(this.source, this.matches)); + return new MarkdownQuery(this.editor().remove(this.matches)); } insertBefore(markdown: Markdown): MarkdownQuery { @@ -199,7 +181,7 @@ export class MarkdownQuery { } insertAfter(markdown: Markdown): MarkdownQuery { - return this.insertAll((range) => blockEnd(range), markdown); + return this.insertAll((range) => MarkdownEditor.blockEnd(range), markdown); } prepend(markdown: Markdown): MarkdownQuery { @@ -212,22 +194,22 @@ export class MarkdownQuery { addRow(row: Record): MarkdownQuery { return new MarkdownQuery( - spliceRanges(this.source, this.matches, (range) => { + this.editor().replace(this.matches, (range) => { if (range.token.type !== 'table') throw new MdqOperationError(`addRow needs a table, got ${range.token.type}`); const table = range.token as Tokens.Table; const headers = table.header.map((cell) => cell.text); const existing = table.rows.map((cells) => headers.map((_, index) => cells[index]?.text || '')); - return renderTable(headers, [...existing, headers.map((header) => row[header] || '')], table.align); + return MarkdownEditor.table(headers, [...existing, headers.map((header) => row[header] || '')], table.align); }) ); } addItem(text: string): MarkdownQuery { return new MarkdownQuery( - spliceRanges(this.source, this.matches, (range) => { + this.editor().replace(this.matches, (range) => { if (range.token.type !== 'list') throw new MdqOperationError(`addItem needs a list, got ${range.token.type}`); const raw = (((range.token as any).raw as string) || '').trimEnd(); - return `${raw}\n${renderItem(raw, text)}\n`; + return `${raw}\n${MarkdownEditor.item(raw, text)}\n`; }) ); } @@ -235,12 +217,12 @@ export class MarkdownQuery { setEntry(key: string, value: string | null): MarkdownQuery { return this.replaceEach((match) => { const token = match.matches[0].token; - return rewriteEntries(this.tokenText(token), key, value, token.type === 'blockquote'); + return MarkdownEditor.entries(this.tokenText(token), key, value, token.type === 'blockquote'); }); } setFrontmatter(key: string, value: unknown): MarkdownQuery { - return new MarkdownQuery(writeFrontmatter(this.source, key, value)); + return new MarkdownQuery(this.editor().setFrontmatter(key, value)); } toString(): string { @@ -251,41 +233,39 @@ export class MarkdownQuery { return this.source; } - /** @deprecated Use text(). */ + /** Compatibility aliases. Each delegates to the canonical name above; prefer those. */ get(): string { return this.text(); } - /** @deprecated Use nodes(). */ meta(): NodeInfo[] { return this.nodes(); } - /** @deprecated Use rows(). */ toJson(): Record[] { return this.rows(); } - /** @deprecated Use entries(). */ keyValue(): Record { return this.entries(); } - /** @deprecated Use setEntry(). */ setKeyValue(key: string, value: string | null): MarkdownQuery { return this.setEntry(key, value); } - /** @deprecated Use preceding(). */ before(): MarkdownQuery { return this.preceding(); } - /** @deprecated Use following(). */ after(): MarkdownQuery { return this.following(); } + private editor(): MarkdownEditor { + return new MarkdownEditor(this.source); + } + private run(candidates: MatchedRange[], segments: QuerySegment[]): MatchedRange[] { if (segments.length === 0) return candidates; @@ -311,9 +291,8 @@ export class MarkdownQuery { } const depth = segment.selector.match(/^h([1-6])$/); - let type = TOKEN_TYPES[segment.selector]; + let type = TOKEN_ALIASES[segment.selector] || segment.selector; if (depth) type = 'heading'; - if (!type) return []; let matches = candidates.filter((range) => range.token.type === type); if (depth) matches = matches.filter((range) => (range.token as any).depth === Number.parseInt(depth[1], 10)); @@ -340,7 +319,7 @@ export class MarkdownQuery { const next = candidates[j]; if (next.token.type === 'heading' && (next.token as Tokens.Heading).depth <= heading.depth) break; innerTokens.push(next); - end = blockEnd(next); + end = MarkdownEditor.blockEnd(next); } sections.push({ token: range.token, start: range.start, length: end - range.start, innerTokens }); @@ -422,8 +401,7 @@ export class MarkdownQuery { if (this.isComment(token)) return (value.raw as string).trim().replace(/^$/, '').trim(); if (token.type === 'html') return value.raw || ''; if (token.type === 'table') return [...(value.header || []).map((cell: any) => cell.text), ...(value.rows || []).flatMap((row: any) => row.map((cell: any) => cell.text))].join(', '); - if (TEXT_TOKENS.has(token.type)) return value.text || ''; - return ''; + return value.text || ''; } private isComment(token: Token): boolean { @@ -432,12 +410,7 @@ export class MarkdownQuery { } private insertAll(offsetOf: (range: MatchedRange) => number, markdown: Markdown): MarkdownQuery { - const offsets = this.matches.map(offsetOf).sort((a, b) => a - b); - let result = this.source; - for (let i = offsets.length - 1; i >= 0; i--) { - result = insertAt(result, offsets[i], String(markdown)); - } - return new MarkdownQuery(result); + return new MarkdownQuery(this.editor().insert(this.matches.map(offsetOf), String(markdown))); } private containerStart(range: MatchedRange): number { @@ -449,7 +422,7 @@ export class MarkdownQuery { if (!range.innerTokens) throw new MdqOperationError(`append needs a section or list, got ${range.token.type}`); const last = range.innerTokens[range.innerTokens.length - 1]; if (!last) return this.containerStart(range); - return blockEnd(last); + return MarkdownEditor.blockEnd(last); } } @@ -518,7 +491,7 @@ export function parseQuery(input: string): QuerySegment[] { } export function buildTokenIndex(source: string): MatchedRange[] { - const { body, offset } = splitFrontmatter(source); + const { body, offset } = MarkdownEditor.splitFrontmatter(source); const ranges: MatchedRange[] = []; let cursor = offset; @@ -537,6 +510,12 @@ export function buildTokenIndex(source: string): MatchedRange[] { return ranges; } +const SELECTOR_NAMES = new Set(['section', 'heading', 'paragraph', 'table', 'list', 'item', 'code', 'blockquote', 'hr', 'html', 'comment']); + +const TOKEN_ALIASES: Record = { item: 'list_item' }; + +const SEGMENT = /(\s*\.?)([A-Za-z]\w*)(?:\((!?)(~?)(?:"((?:[^"\\]|\\.)*)"|\/((?:[^/\\]|\\.)*)\/([a-z]*))\))?((?:\[[^\]]*\])*)\s*/y; + export type Markdown = string | MarkdownQuery; export type Matcher = string | RegExp | ((text: string) => boolean); diff --git a/tests/unit/mdq/frontmatter.test.ts b/tests/unit/mdq/frontmatter.test.ts index 1cc7c150..65e9e1d9 100644 --- a/tests/unit/mdq/frontmatter.test.ts +++ b/tests/unit/mdq/frontmatter.test.ts @@ -1,24 +1,24 @@ import { describe, expect, it } from 'vitest'; -import { splitFrontmatter } from '../../../src/utils/mdq/edit.ts'; +import { MarkdownEditor } from '../../../src/utils/mdq/edit.ts'; import { buildTokenIndex, mdq } from '../../../src/utils/mdq/query.ts'; describe('splitFrontmatter', () => { it('splits a leading yaml block from the body', () => { const src = '---\nurl: /login\nwait: 1000\n---\n\n# Title\n'; - const fm = splitFrontmatter(src); + const fm = MarkdownEditor.splitFrontmatter(src); expect(fm.raw).toBe('url: /login\nwait: 1000'); expect(fm.body).toBe('\n# Title\n'); expect(fm.offset).toBe(src.length - fm.body.length); }); it('returns no frontmatter when the document does not open with ---', () => { - const fm = splitFrontmatter('# Title\n\n---\n'); + const fm = MarkdownEditor.splitFrontmatter('# Title\n\n---\n'); expect(fm.raw).toBe(''); expect(fm.offset).toBe(0); }); it('treats an unterminated --- as body, not frontmatter', () => { - const fm = splitFrontmatter('---\nnot closed\n'); + const fm = MarkdownEditor.splitFrontmatter('---\nnot closed\n'); expect(fm.raw).toBe(''); expect(fm.offset).toBe(0); });