diff --git a/.github/workflows/publish-mdq.yml b/.github/workflows/publish-mdq.yml new file mode 100644 index 00000000..5abd5fc8 --- /dev/null +++ b/.github/workflows/publish-mdq.yml @@ -0,0 +1,97 @@ +name: Publish mdq to npm + +on: + release: + types: [published] + +jobs: + publish: + if: startsWith(github.event.release.tag_name, 'mdq-') + runs-on: ubuntu-latest + + permissions: + contents: read + id-token: write + + steps: + - name: Checkout the released commit + uses: actions/checkout@v4 + with: + ref: ${{ github.event.release.tag_name }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 24 + registry-url: https://registry.npmjs.org + + - name: Upgrade npm for tokenless (OIDC) publishing + run: npm install -g npm@latest + + - name: Setup Bun + uses: oven-sh/setup-bun@v1 + with: + bun-version: latest + + - name: Install dependencies + run: bun install + + - name: Run mdq tests + run: bun test tests/unit/mdq/ + + - name: Set version from release + id: release + env: + TAG: ${{ github.event.release.tag_name }} + PRERELEASE: ${{ github.event.release.prerelease }} + run: | + VERSION="${TAG#mdq-}" + VERSION="${VERSION#v}" + echo "Publishing mdq $VERSION from release $TAG" + npm version "$VERSION" --no-git-tag-version + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + if [[ "$PRERELEASE" == "true" || "$VERSION" == *beta* || "$VERSION" == *pre* || "$VERSION" == *alpha* || "$VERSION" == *rc* ]]; then + echo "tag=beta" >> "$GITHUB_OUTPUT" + else + echo "tag=latest" >> "$GITHUB_OUTPUT" + fi + + - name: Stage mdq package + run: bun run build:mdq + + - name: Verify mdq tarball on Node.js + run: | + npm pack ./dist-mdq --pack-destination /tmp + mkdir -p /tmp/mdq-smoke && cd /tmp/mdq-smoke + npm init -y > /dev/null + npm install --ignore-scripts --no-audit --no-fund /tmp/mdq-*.tgz + printf '# Doc\n\n## API\n\n| Method | Path |\n|---|---|\n| GET | /users |\n' > sample.md + + echo "CLI, file argument" + ./node_modules/.bin/mdq 'h2' sample.md + + echo "CLI, stdin" + cat sample.md | ./node_modules/.bin/mdq 'section("API") table' --json + + echo "CLI, file argument while stdin is an open pipe (must not EAGAIN)" + ( sleep 3 ) | ./node_modules/.bin/mdq 'h2' sample.md + + echo "CLI, exit code 1 when nothing matches" + ./node_modules/.bin/mdq 'h5' sample.md && exit 1 || test $? -eq 1 + + echo "Library import" + node --input-type=module -e " + import { mdq } from 'mdq'; + const doc = mdq('---\nurl: /x\n---\n\n## A\n\n- one\n'); + if (doc.frontmatter().url !== '/x') throw new Error('frontmatter'); + if (doc.query('h2').count() !== 1) throw new Error('headings'); + if (!doc.list().addItem('two').toString().includes('- two')) throw new Error('addItem'); + " + + - name: Publish mdq to npm + run: | + if npm view "mdq@${{ steps.release.outputs.version }}" version > /dev/null 2>&1; then + echo "mdq@${{ steps.release.outputs.version }} is already published, skipping" + exit 0 + fi + npm publish ./dist-mdq --access public --tag ${{ steps.release.outputs.tag }} diff --git a/.gitignore b/.gitignore index 92f72598..3afdcf65 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ session.json # Build outputs dist/ dist-prima/ +dist-mdq/ build/ out/ .types-build/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 706c2399..de793429 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,63 @@ ## 2026-09-15 +### New CLI Options + +- **`mdq`** — A new command for reading and editing markdown from the shell, the way `jq` reads JSON. + It ships as its own npm package, so it runs without installing anything: `npx mdq 'h2' README.md`. + It also works as a library — `import { mdq } from 'mdq'` — on Node 18 or newer. + The first argument is a selector, the second an optional file (stdin is used when it is omitted). + Matched markdown is printed by default. Exit codes compose like `grep`: `0` when something matched, + `1` when nothing did, `2` for a bad selector or bad usage. + + ```bash + mdq 'h2' README.md # print every h2 + cat plan.md | mdq 'section("API") table' # read from stdin + mdq 'comment(~"test")' plan.md --count # how many test comments + ``` + +- **`--json`, `--count`, `--text`, `--frontmatter`** — Change what is printed: table rows as JSON, + the number of matches, the text with its markdown stripped, or the file's frontmatter as JSON. + + ```bash + mdq 'section("API") table' --json README.md + mdq 'h2' --count README.md + mdq --frontmatter knowledge/login.md + ``` + +- **`--remove`, `--replace`, `--insert-before`, `--insert-after`, `--prepend`, `--append`, `--add-row`, `--add-item`, `--set`** — + Edit the file instead of reading it. The whole document is printed with the edit applied; one edit at a + time. `--add-row` takes a JSON object and lines the table's columns back up; `--add-item` copies the + list's existing bullet or numbering; `--set` takes `Key=value` and drops the entry when the value is empty. + + ```bash + mdq 'section("FAQ")' doc.md --remove + mdq 'table[0]' api.md --add-row '{"Method":"GET","Path":"/users"}' + mdq 'list' plan.md --add-item 'check the dashboard' + ``` + +- **`-i`, `--in-place`** — Write the edit back to the file instead of printing it. + + ```bash + mdq 'section("Draft")' notes.md --remove -i + ``` + +### Changes + +- Knowledge and experience files are now read correctly when they start with a `---` block. The + frontmatter used to be read as a heading titled `url: /login`, so the first fact in every such file + could be mistaken for a section title. +- Blank lines are no longer lost or doubled when a block is deleted or inserted, and blank lines inside + fenced code blocks are left exactly as they were. +- A mistyped selector now fails with a message naming the unknown word, instead of quietly matching + nothing and looking like an empty file. +- A search written as `/pattern/` is now case-sensitive unless it ends with `i`, matching how quoted + searches already behaved. Previously every `/pattern/` ignored case whether it said so or not. +- Searching a table now looks at its cells, not only its column titles. +- Reading a file no longer waits on standard input. Passing a filename while standard input was an + open pipe used to abort with an `EAGAIN` error on Node; input is now read only when no filename + is given. + ### Fixes - [Pilot] A control the page has disabled is no longer read as one missing required field. Pilot had a diff --git a/bin/mdq.ts b/bin/mdq.ts new file mode 100644 index 00000000..d8e03608 --- /dev/null +++ b/bin/mdq.ts @@ -0,0 +1,18 @@ +#!/usr/bin/env node +import { runMdq } from '../src/utils/mdq/cli.ts'; + +async function readStdin(): Promise { + if (process.stdin.isTTY) return ''; + process.stdin.setEncoding('utf8'); + let text = ''; + for await (const chunk of process.stdin) text += chunk; + return text; +} + +const result = await runMdq(process.argv.slice(2), readStdin); +if (result.output) { + let text = result.output; + if (!text.endsWith('\n')) text = `${text}\n`; + process.stdout.write(text); +} +process.exit(result.code); diff --git a/boat/prima/src/prima.ts b/boat/prima/src/prima.ts index bb7a06e8..fc489d16 100644 --- a/boat/prima/src/prima.ts +++ b/boat/prima/src/prima.ts @@ -57,7 +57,8 @@ function dropVolatileColumns(markdown: string): string { const divider = `|${columns.map(() => '------').join('|')}|`; const body = rows.map((row) => `| ${columns.map((name) => row[name] || '-').join(' | ')} |`); return [header, divider, ...body, ''].join('\n'); - }); + }) + .toString(); } function cap(text: string, max: number): string { diff --git a/docs/superpowers/plans/2026-09-15-mdq-package.md b/docs/superpowers/plans/2026-09-15-mdq-package.md index ecedbe44..fc4a0ac8 100644 --- a/docs/superpowers/plans/2026-09-15-mdq-package.md +++ b/docs/superpowers/plans/2026-09-15-mdq-package.md @@ -1,6 +1,6 @@ # mdq Package Implementation Plan -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [x]`) syntax for tracking. **Goal:** Extract `src/utils/markdown-query.ts` into `src/utils/mdq/` as a publish-ready package that can both query and update markdown, then add a jq-like CLI. @@ -68,7 +68,7 @@ Move the parser to its new home and teach it the one thing it gets wrong today: The `trailing` field is new and load-bearing: `marked` emits `space` tokens as siblings (a `paragraph` raw is `"para"` with no newline, followed by a separate `space` raw of `"\n\n"`), so every write verb needs to know where a node's separator lives. -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // tests/unit/mdq/frontmatter.test.ts @@ -127,12 +127,12 @@ describe('buildTokenIndex', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/frontmatter.test.ts` Expected: FAIL — cannot resolve `src/utils/mdq/query.ts` -- [ ] **Step 3: Create query.ts with the index** +- [x] **Step 3: Create query.ts with the index** Copy `src/utils/markdown-query.ts` to `src/utils/mdq/query.ts` verbatim first, then apply these three changes. @@ -207,12 +207,12 @@ export interface MatchedRange { } ``` -- [ ] **Step 4: Run test to verify it passes** +- [x] **Step 4: Run test to verify it passes** Run: `bun test tests/unit/mdq/frontmatter.test.ts` Expected: PASS (8 tests) -- [ ] **Step 5: Format, lint and commit** +- [x] **Step 5: Format, lint and commit** ```bash bun run format && bun run lint:fix @@ -236,7 +236,7 @@ Get every existing call site running against the new file with **zero behaviour - Consumes: `buildTokenIndex`, `splitFrontmatter`, `MatchedRange` from Task 1 - Produces: `mdq(source: string): MarkdownQuery`, class `MarkdownQuery`, `parseQuery`, all existing methods unchanged -- [ ] **Step 1: Move the test file and repoint its import** +- [x] **Step 1: Move the test file and repoint its import** ```bash git mv tests/unit/markdown-query.test.ts tests/unit/mdq/query.test.ts @@ -254,12 +254,12 @@ to: import { mdq, parseQuery } from '../../../src/utils/mdq/query.ts'; ``` -- [ ] **Step 2: Run the suite to verify it fails** +- [x] **Step 2: Run the suite to verify it fails** Run: `bun test tests/unit/mdq/query.test.ts` Expected: FAIL — `mdq` / `parseQuery` are not yet exported from `query.ts`, or section tests fail because `expandSectionRanges` still assumes `space` tokens are present -- [ ] **Step 3: Restore the full API in query.ts** +- [x] **Step 3: Restore the full API in query.ts** Everything from the original `markdown-query.ts` below `buildTokenIndex` — `matchText`, `entryKey`, `getTokenText`, `getHeadingDepth`, `isSectionSelector`, `getSectionDepth`, `selectorToTokenType`, `computeSections`, `extractListItems`, `applyIndexSlice`, `expandSectionRanges`, `executeSegments`, `class MarkdownQuery`, `mdq` — carries over unchanged, except: @@ -277,23 +277,23 @@ for (let j = i + 1; j < candidates.length; j++) { Move every `export interface` / `export type` to the end of the file, and replace the ternaries at the original lines 30, 34, 129-130 and 295 with early returns. -- [ ] **Step 4: Run the suite to verify it passes** +- [x] **Step 4: Run the suite to verify it passes** Run: `bun test tests/unit/mdq/query.test.ts` Expected: PASS — **110 tests**, the same count as before the move -- [ ] **Step 5: Replace markdown-query.ts with a shim** +- [x] **Step 5: Replace markdown-query.ts with a shim** ```ts export * from './mdq/query.ts'; ``` -- [ ] **Step 6: Verify every existing call site still works** +- [x] **Step 6: Verify every existing call site still works** Run: `bun test tests/unit/` Expected: PASS, no new failures versus the pre-task run -- [ ] **Step 7: Format, lint and commit** +- [x] **Step 7: Format, lint and commit** ```bash bun run format && bun run lint:fix @@ -327,7 +327,7 @@ The one risky task. It ends with the repo green and every break fixed. - `mdq(source: Markdown): MarkdownDoc` - Deprecated alias `MarkdownQuery = Selection` -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // append to tests/unit/mdq/query.test.ts @@ -359,12 +359,12 @@ describe('MarkdownDoc chaining', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/query.test.ts -t 'MarkdownDoc chaining'` Expected: FAIL — `.query is not a function` on the string returned by `replace` -- [ ] **Step 3: Split MarkdownQuery into MarkdownDoc and Selection** +- [x] **Step 3: Split MarkdownQuery into MarkdownDoc and Selection** `MarkdownDoc` holds the source. `Selection` holds source plus matches. Every write on `Selection` ends by wrapping its result: @@ -411,6 +411,9 @@ export function mdq(source: Markdown): MarkdownDoc { } ``` +`query` takes only a selector here. The optional second `matcher` argument arrives in +Task 5 — it is not missing. + At the end of the file: ```ts @@ -420,12 +423,20 @@ export type Markdown = string | MarkdownDoc; export const MarkdownQuery = Selection; ``` -- [ ] **Step 4: Run test to verify it passes** +Before relying on that alias, confirm nothing imports the class by name: + +```bash +git grep -ln "MarkdownQuery" -- src bin boat tests +``` + +Expected: only `src/utils/mdq/query.ts`. Any other file needs its import checked. + +- [x] **Step 4: Run test to verify it passes** Run: `bun test tests/unit/mdq/query.test.ts -t 'MarkdownDoc chaining'` Expected: PASS (4 tests) -- [ ] **Step 5: Fix the write assertions in the ported suite** +- [x] **Step 5: Fix the write assertions in the ported suite** The existing `replace` tests assert against a string. Wrap each in `String(...)`, for example: @@ -442,7 +453,7 @@ Apply the same to every assertion in the `replace`, `setKeyValue` and `edge case Run: `bun test tests/unit/mdq/query.test.ts` Expected: PASS — 114 tests -- [ ] **Step 6: Migrate the four breakage classes** +- [x] **Step 6: Migrate the four breakage classes** **(a) Assignment into a `string`-typed target** — append `.toString()`: @@ -490,7 +501,7 @@ Expected: PASS — 114 tests **Leave alone** — these already work because `mdq()` accepts a `MarkdownDoc`: `planner.ts:303`, `planner.ts:405`. -- [ ] **Step 7: Fix the one regex call site** +- [x] **Step 7: Fix the one regex call site** `src/ai/researcher.ts:316` relies on regex matching being implicitly case-insensitive. Task 4 removes that. Make the flag explicit now so the two changes never overlap: @@ -498,27 +509,33 @@ Expected: PASS — 114 tests const summaryText = mdq(result.text).query('section2(/^summary/i)').query('paragraph[0]').text().trim(); ``` -- [ ] **Step 8: Verify with the scoped type check** +- [x] **Step 8: Verify with the scoped type check** + +Grep by **type name, repo-wide** — not by file path. A path-scoped grep misses test +helpers and bin scripts, and those break too: ```bash -bunx tsc -p tsconfig.json --noEmit 2>&1 | grep -E "^(src/utils/mdq/|src/utils/markdown-query|src/experience-tracker|src/ai/planner|src/ai/researcher)" +bunx tsc -p tsconfig.json --noEmit 2>&1 | grep -E "MarkdownDoc|MarkdownQuery|Selection" ``` -Expected: **exactly these two lines and nothing else.** +Expected: **no output at all.** -``` -src/ai/researcher/locators.ts(247,41): error TS2339: Property 'playwrightLocatorCount' does not exist on type 'Explorer'. -src/ai/researcher/locators.ts(247,65): error TS7006: Parameter 'page' implicitly has an 'any' type. -``` +Any line is an unmigrated call site. CI will not catch it, because CI runs `tsc --noCheck`. -Any third line is an unmigrated call site. Fix it before continuing — CI will not catch it, because CI runs `tsc --noCheck`. +The audit in Step 6 lists eleven sites; the real count is **seventeen**. The extras are all +the same shape — a function whose declared return type is `string` now returns a +`MarkdownDoc` — in `boat/prima/src/prima.ts`, `planner.ts` (three chains), +`deep-analysis.ts:542` and `experience-tracker.ts:511`, plus a test helper in +`tests/unit/research-parser-pagination.test.ts` that feeds the result to `marked.lexer`, +which does **not** coerce and throws `e.replace is not a function` at runtime. Trust the +grep, not the list. -- [ ] **Step 9: Run the full unit suite** +- [x] **Step 9: Run the full unit suite** Run: `bun test tests/unit/` Expected: PASS, no new failures -- [ ] **Step 10: Format, lint and commit** +- [x] **Step 10: Format, lint and commit** ```bash bun run format && bun run lint:fix @@ -540,7 +557,7 @@ Four grammar changes, all additive now that Task 3 pre-fixed the one regex call - Consumes: `parseQuery`, `getTokenText`, `selectorToTokenType`, `matchText` from Task 2 - Produces: `class MdqError extends Error`, `class MdqSelectorError extends MdqError` (with `index: number`), selectors `comment` and `html` -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // tests/unit/mdq/selectors.test.ts @@ -629,7 +646,7 @@ describe('selector errors', () => { mdq(doc).query('h2("A") secton("B")'); expect.unreachable(); } catch (error) { - expect(error.index).toBe(7); + expect(error.index).toBe(8); } }); @@ -639,12 +656,12 @@ describe('selector errors', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/selectors.test.ts` Expected: FAIL — `MdqSelectorError` is not exported -- [ ] **Step 3: Implement the four changes** +- [x] **Step 3: Implement the four changes** Error classes, at the top of the class section: @@ -769,17 +786,17 @@ export interface NodeInfo { } ``` -- [ ] **Step 4: Run test to verify it passes** +- [x] **Step 4: Run test to verify it passes** Run: `bun test tests/unit/mdq/selectors.test.ts` Expected: PASS (13 tests) -- [ ] **Step 5: Verify nothing regressed** +- [x] **Step 5: Verify nothing regressed** Run: `bun test tests/unit/` Expected: PASS — in particular `query.test.ts` still at 114, since Task 3 already fixed `researcher.ts:316` -- [ ] **Step 6: Format, lint and commit** +- [x] **Step 6: Format, lint and commit** ```bash bun run format && bun run lint:fix @@ -806,7 +823,7 @@ Removes the hand-escaping wart: `section.name.replace(/"/g, '\\"')` at `research - Sugar on both classes: `section` `heading` `paragraph` `table` `list` `item` `code` `blockquote` `comment` `html` `hr` - `at(n: number): Selection` and `slice(from?: number, to?: number): Selection` on `Selection` -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // tests/unit/mdq/sugar.test.ts @@ -918,12 +935,12 @@ describe('canonical read names', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/sugar.test.ts` Expected: FAIL — `mdq(...).heading is not a function` -- [ ] **Step 3: Implement matchers and the shared sugar base** +- [x] **Step 3: Implement matchers and the shared sugar base** A value matcher bypasses the grammar entirely, so it needs its own `TextMatcher` mode: @@ -1027,7 +1044,7 @@ On `Selection`, add: At the end of the file add `Matcher`, `SelectorOptions`, `predicate?: (text: string) => boolean` on `TextMatcher`, and widen its `mode` to include `'predicate'`. -- [ ] **Step 4: Run test to verify it passes** +- [x] **Step 4: Run test to verify it passes** Run: `bun test tests/unit/mdq/sugar.test.ts` Expected: PASS (21 tests) @@ -1035,7 +1052,7 @@ Expected: PASS (21 tests) The `canonical read names` block fails until Step 5 adds the renames — that is expected. Run Step 5 before treating those four as real failures. -- [ ] **Step 5: Add the read renames and their deprecated aliases** +- [x] **Step 5: Add the read renames and their deprecated aliases** Canonical names, with the old ones kept and marked: @@ -1060,7 +1077,7 @@ Each alias is one line, for example: Run: `bun test tests/unit/` Expected: PASS — the ported suite still calls the deprecated names and must keep working -- [ ] **Step 6: Format, lint and commit** +- [x] **Step 6: Format, lint and commit** ```bash bun run format && bun run lint:fix @@ -1093,7 +1110,7 @@ This matters because `marked` separators are uneven: a `heading` raw is `"# A\n\ - Produces, on `Selection`: `remove()`, `insertBefore(md)`, `insertAfter(md)`, `prepend(md)`, `append(md)` - Produces, on `MarkdownDoc`: `append(md)`, `prepend(md)` -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // tests/unit/mdq/edit.test.ts @@ -1199,12 +1216,12 @@ describe('chained edits', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/edit.test.ts` Expected: FAIL — `.remove is not a function` -- [ ] **Step 3: Write edit.ts** +- [x] **Step 3: Write edit.ts** ```ts import type { MatchedRange } from './query.ts'; @@ -1238,20 +1255,28 @@ export function removeRanges(source: string, ranges: MatchedRange[]): string { let result = source; for (let i = ordered.length - 1; i >= 0; i--) { const range = ordered[i]; - const end = blockEnd(range); - let start = range.start; - if (!range.trailing) start = trimPrecedingBlankLine(result, start); - result = result.slice(0, start) + result.slice(end); + const head = result.slice(0, range.start); + const tail = result.slice(blockEnd(range)); + if (tail) { + result = head + tail; + continue; + } + if (!head) { + result = ''; + continue; + } + result = `${head.replace(/\n+$/, '')}\n`; } return result; } export function insertAt(source: string, offset: number, markdown: string): string { const block = normalizeBlock(markdown); - const before = source.slice(0, offset); - const after = source.slice(offset); - if (!after) return `${before}${before.endsWith('\n') ? '' : '\n'}\n${block}`.replace(/\n{3,}/g, '\n\n'); - return `${before}${block}\n${after}`.replace(/\n{3,}/g, '\n\n'); + const before = source.slice(0, offset).replace(/\n+$/, ''); + const after = source.slice(offset).replace(/^\n+/, ''); + if (!before) return `${block}\n${after}`; + if (!after) return `${before}\n\n${block}`; + return `${before}\n\n${block}\n${after}`; } function dedupeRanges(ranges: MatchedRange[]): MatchedRange[] { @@ -1266,17 +1291,26 @@ function dedupeRanges(ranges: MatchedRange[]): MatchedRange[] { return kept; } -function trimPrecedingBlankLine(source: string, start: number): number { - let cursor = start; - while (cursor > 0 && source[cursor - 1] === '\n') cursor--; - if (cursor === 0) return 0; - return cursor + 1; -} ``` -Note `insertAt` collapses any run of three or more newlines to exactly two. That single rule is what enforces the invariant across every insert path, rather than each verb reasoning about separators itself. +Both rules below were derived from real `marked` output, not assumed. **Do not "simplify" +either one.** + +**`insertAt` normalizes only the seam.** It strips newlines from the end of `before` and +the start of `after`, then rebuilds the join. The tempting alternative — a global +`.replace(/\n{3,}/g, '\n\n')` over the document — is wrong: a fenced code block's raw really +does contain runs of blank lines (`marked` lexes a ```js block holding `a\n\n\nb` as one +`code` token whose raw carries `\n\n\n`), so a global collapse silently rewrites user code. + +**`removeRanges` collapses only at end-of-document.** `marked` bakes the separator into +some raws and not others: a mid-document `paragraph` raw is `"para"` with a sibling `space` +token, a document-final `paragraph` raw is `"last\n"` with no sibling, and every `heading` +raw carries its own `"\n\n"`. So "no trailing space, therefore trim backwards" is wrong — on +`'# A\n\n## B\n\ntext\n'` it eats a blank line and yields `'# A\ntext\n'`. Deleting +`[start, blockEnd)` is already correct whenever anything follows; only a node removed from +the very end needs repair. -- [ ] **Step 4: Wire the verbs onto Selection and MarkdownDoc** +- [x] **Step 4: Wire the verbs onto Selection and MarkdownDoc** On `Selection`, four public verbs delegating to one private helper: @@ -1355,14 +1389,14 @@ export class MdqOperationError extends MdqError { } ``` -- [ ] **Step 5: Run test to verify it passes** +- [x] **Step 5: Run test to verify it passes** Run: `bun test tests/unit/mdq/edit.test.ts` Expected: PASS (19 tests) If a whitespace assertion fails, print the actual output with `JSON.stringify` before changing anything — the difference is almost always one newline, and guessing at it will break a different case. -- [ ] **Step 6: Run the whole suite and commit** +- [x] **Step 6: Run the whole suite and commit** ```bash bun test tests/unit/ @@ -1385,7 +1419,7 @@ git commit -m "feat(mdq): remove and insert verbs with whitespace normalization" - Produces in `edit.ts`: `renderTable(headers: string[], rows: string[][], align: (string | null)[]): string`, `renderItem(listRaw: string, text: string): string` - Produces on `Selection`: `addRow(row: Record): MarkdownDoc`, `addItem(text: string): MarkdownDoc` -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // tests/unit/mdq/structural.test.ts @@ -1451,12 +1485,12 @@ describe('addItem', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/structural.test.ts` Expected: FAIL — `.addRow is not a function` -- [ ] **Step 3: Add the renderers to edit.ts** +- [x] **Step 3: Add the renderers to edit.ts** ```ts export function renderTable(headers: string[], rows: string[][], align: (string | null)[]): string { @@ -1484,7 +1518,7 @@ function dashes(alignment: string | null, width: number): string { } ``` -- [ ] **Step 4: Wire the verbs onto Selection** +- [x] **Step 4: Wire the verbs onto Selection** ```ts addRow(row: Record): MarkdownDoc { @@ -1510,12 +1544,12 @@ function dashes(alignment: string | null, width: number): string { } ``` -- [ ] **Step 5: Run test to verify it passes** +- [x] **Step 5: Run test to verify it passes** Run: `bun test tests/unit/mdq/structural.test.ts` Expected: PASS (10 tests) -- [ ] **Step 6: Run the whole suite and commit** +- [x] **Step 6: Run the whole suite and commit** ```bash bun test tests/unit/ @@ -1541,7 +1575,7 @@ git commit -m "feat(mdq): addRow and addItem structural inserts" Reading and writing both go through `yaml`'s **Document API** (`YAML.parseDocument`), never `parse`/`stringify`. That is what preserves comments through a write — verified behaviour, not an assumption. -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // append to tests/unit/mdq/frontmatter.test.ts @@ -1612,19 +1646,19 @@ describe('entries and setEntry', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/frontmatter.test.ts` Expected: FAIL — `.frontmatter is not a function` -- [ ] **Step 3: Implement in edit.ts** +- [x] **Step 3: Implement in edit.ts** ```ts import YAML from 'yaml'; export function writeFrontmatter(source: string, key: string, value: unknown): string { const { raw, body, offset } = splitFrontmatter(source); - const document = YAML.parseDocument(raw || ''); + const document = raw ? YAML.parseDocument(raw) : new YAML.Document({}); if (value === null) document.delete(key); if (value !== null) document.set(key, value); const rendered = document.toString().replace(/\s+$/, ''); @@ -1662,7 +1696,7 @@ export function rewriteEntries(tokenText: string, isBlockquote: boolean, key: st `entryKey` moves to `edit.ts` alongside it. -- [ ] **Step 4: Wire onto the classes** +- [x] **Step 4: Wire onto the classes** ```ts frontmatter(): Record { @@ -1685,12 +1719,12 @@ export function rewriteEntries(tokenText: string, isBlockquote: boolean, key: st } ``` -- [ ] **Step 5: Run test to verify it passes** +- [x] **Step 5: Run test to verify it passes** Run: `bun test tests/unit/mdq/frontmatter.test.ts` Expected: PASS (21 tests) -- [ ] **Step 6: Confirm knowledge and experience files now parse correctly** +- [x] **Step 6: Confirm knowledge and experience files now parse correctly** This is the real-world check that motivated the feature: @@ -1710,7 +1744,7 @@ Expected: frontmatter parsed as an object on each file, and **no heading whose t If either directory is empty, skip this step and note it. -- [ ] **Step 7: Run the whole suite and commit** +- [x] **Step 7: Run the whole suite and commit** ```bash bun test tests/unit/ @@ -1732,7 +1766,7 @@ The package is publish-ready only if someone can use it without reading the sour - Consumes: the complete API from Tasks 3-8 - Produces: nothing code depends on -- [ ] **Step 1: Write the README** +- [x] **Step 1: Write the README** Cover, in this order: @@ -1750,7 +1784,7 @@ Do **not** document the deprecated aliases (`get` `toJson` `keyValue` `setKeyVal Follow the repo docs style: show each format example once, and do not close with a "Why this matters" section. -- [ ] **Step 2: Verify every example in the README actually runs** +- [x] **Step 2: Verify every example in the README actually runs** Extract each fenced `js` block and execute it. Any example that throws or prints something other than what the README claims is a documentation bug — fix the README, not the test. @@ -1761,7 +1795,7 @@ import { mdq } from "./src/utils/mdq/query.ts"; ' ``` -- [ ] **Step 3: Commit** +- [x] **Step 3: Commit** ```bash git add src/utils/mdq/README.md @@ -1786,7 +1820,7 @@ git commit -m "docs(mdq): package README" Note a deliberate deviation from `CLAUDE.md`: command logic normally lives in `src/commands/`, but mdq must not import from anywhere in explorbot. Its CLI ships with the package. -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** ```ts // tests/unit/mdq/cli.test.ts @@ -1853,7 +1887,9 @@ describe('edits', () => { }); it('sets an entry', async () => { - expect((await runMdq(['blockquote', '--set', 'Container=.x'], '> Container: .old\n')).output).toContain('.x'); + const result = await runMdq(['blockquote', '--set', 'Container=.x'], '> Container: .old\n'); + expect(result.output).toContain('.x'); + expect(result.output).not.toContain('.old'); }); }); @@ -1874,12 +1910,12 @@ describe('exit codes', () => { }); ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: `bun test tests/unit/mdq/cli.test.ts` Expected: FAIL — cannot resolve `cli.ts` -- [ ] **Step 3: Implement cli.ts** +- [x] **Step 3: Implement cli.ts** Use Commander with `exitOverride()` and `.configureOutput()` so a parse failure surfaces as a return value rather than killing the process. Shape: @@ -1926,12 +1962,12 @@ export interface CliResult { } ``` -- [ ] **Step 4: Run test to verify it passes** +- [x] **Step 4: Run test to verify it passes** Run: `bun test tests/unit/mdq/cli.test.ts` Expected: PASS (13 tests) -- [ ] **Step 5: Add the thin bin entry** +- [x] **Step 5: Add the thin bin entry** ```ts #!/usr/bin/env bun @@ -1950,7 +1986,7 @@ Add to `package.json` `bin`: "mdq": "./dist/bin/mdq.js" ``` -- [ ] **Step 6: Smoke-test the real binary** +- [x] **Step 6: Smoke-test the real binary** ```bash echo '# A @@ -1971,7 +2007,7 @@ bun run bin/mdq.ts 'nonsense' CLAUDE.md; echo "exit=$?" ``` Expected: `Unknown selector "nonsense"` and `exit=2` -- [ ] **Step 7: Run the whole suite and commit** +- [x] **Step 7: Run the whole suite and commit** ```bash bun test tests/unit/ @@ -1987,12 +2023,12 @@ git commit -m "feat(mdq): jq-like CLI" **Files:** - Modify: `CHANGELOG.md` -- [ ] **Step 1: Run the full unit suite** +- [x] **Step 1: Run the full unit suite** Run: `bun test tests/unit/` Expected: PASS, no failures -- [ ] **Step 2: Re-run the scoped type check** +- [x] **Step 2: Re-run the scoped type check** ```bash bunx tsc -p tsconfig.json --noEmit 2>&1 | grep -E "^(src/utils/mdq/|src/utils/markdown-query|src/experience-tracker|src/ai/planner|src/ai/researcher|bin/mdq)" @@ -2000,7 +2036,7 @@ bunx tsc -p tsconfig.json --noEmit 2>&1 | grep -E "^(src/utils/mdq/|src/utils/ma Expected: **exactly the two known `locators.ts(247,...)` lines.** Anything else is a real defect that CI will not catch. -- [ ] **Step 3: Confirm the package has no explorbot imports** +- [x] **Step 3: Confirm the package has no explorbot imports** ```bash grep -rn "^import\|from '" src/utils/mdq/*.ts | grep -v "'marked'" | grep -v "'yaml'" | grep -v "'commander'" | grep -v "'./" @@ -2008,11 +2044,11 @@ grep -rn "^import\|from '" src/utils/mdq/*.ts | grep -v "'marked'" | grep -v "'y Expected: **no output.** Any line here breaks extractability, which is the whole point of the package. -- [ ] **Step 4: Update the changelog** +- [x] **Step 4: Update the changelog** Use the `/changelog` skill, per `CLAUDE.md`. -- [ ] **Step 5: Commit** +- [x] **Step 5: Commit** ```bash git add CHANGELOG.md diff --git a/package.json b/package.json index e122d668..b75e5d9d 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,8 @@ "check": "biome check .", "check:fix": "biome check --write .", "langfuse:export": "bun run .claude/skills/explorbot-debug/langfuse-export.ts", - "build:prima": "bun run scripts/build-prima-npm.ts" + "build:prima": "bun run scripts/build-prima-npm.ts", + "build:mdq": "bun run scripts/build-mdq-npm.ts" }, "keywords": ["cli", "react", "ink", "codeceptjs", "playwright"], "repository": { diff --git a/scripts/build-mdq-npm.ts b/scripts/build-mdq-npm.ts new file mode 100644 index 00000000..f4670579 --- /dev/null +++ b/scripts/build-mdq-npm.ts @@ -0,0 +1,69 @@ +#!/usr/bin/env bun +import { spawnSync } from 'node:child_process'; +import { cpSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; + +const ROOT = process.cwd(); +const STAGE = path.join(ROOT, 'dist-mdq'); +const PACKAGE = path.join(ROOT, 'src', 'utils', 'mdq'); +const EXTERNAL = ['marked', 'yaml', 'commander']; + +rmSync(STAGE, { recursive: true, force: true }); +mkdirSync(path.join(STAGE, 'bin'), { recursive: true }); + +function run(command: string, args: string[]) { + const result = spawnSync(command, args, { encoding: 'utf8' }); + if (result.status === 0) return; + console.error(result.stderr || result.stdout); + process.exit(1); +} + +function bundle(entry: string, outfile: string) { + const args = [entry, '--target', 'node', '--format', 'esm', '--outfile', outfile]; + for (const name of EXTERNAL) args.push('--external', name); + run('bun', ['build', ...args]); +} + +bundle(path.join(PACKAGE, 'query.ts'), path.join(STAGE, 'index.js')); +bundle(path.join(ROOT, 'bin', 'mdq.ts'), path.join(STAGE, 'bin', 'mdq.js')); + +const cli = path.join(STAGE, 'bin', 'mdq.js'); +const shebanged = readFileSync(cli, 'utf8').replace(/^#!.*\n/, ''); +writeFileSync(cli, `#!/usr/bin/env node\n${shebanged}`, { mode: 0o755 }); + +const types = path.join(STAGE, 'types'); +run('bunx', [ + 'tsc', + path.join(PACKAGE, 'query.ts'), + path.join(PACKAGE, 'edit.ts'), + '--declaration', + '--emitDeclarationOnly', + '--noCheck', + '--module', + 'esnext', + '--moduleResolution', + 'bundler', + '--target', + 'esnext', + '--allowImportingTsExtensions', + '--rewriteRelativeImportExtensions', + '--skipLibCheck', + '--outDir', + types, +]); + +for (const file of readdirSync(types)) { + const target = path.join(types, file); + writeFileSync(target, readFileSync(target, 'utf8').replaceAll("./edit.ts'", "./edit.js'")); +} + +const root = JSON.parse(readFileSync(path.join(ROOT, 'package.json'), 'utf8')); +const manifest = JSON.parse(readFileSync(path.join(PACKAGE, 'package.json'), 'utf8')); +manifest.version = root.version; +manifest.dependencies = {}; +for (const name of EXTERNAL) manifest.dependencies[name] = root.dependencies[name]; + +writeFileSync(path.join(STAGE, 'package.json'), `${JSON.stringify(manifest, null, 2)}\n`); +cpSync(path.join(PACKAGE, 'README.md'), path.join(STAGE, 'README.md')); + +console.log(`mdq ${manifest.version} staged in dist-mdq/`); diff --git a/src/ai/planner.ts b/src/ai/planner.ts index 5838da46..8f52b847 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -301,13 +301,14 @@ export class Planner extends PlannerBase implements Agent { .replaceEach((section) => { const heading = section.query('heading').text().trim(); const withoutHeadings = mdq(section.text()).query('heading').replace(''); - const body = mdq(withoutHeadings).query('hr').replace('').trim(); + const body = mdq(withoutHeadings).query('hr').replace('').toString().trim(); if (body && !seenTitles.has(heading)) { seenTitles.add(heading); return section.text(); } return ''; - }); + }) + .toString(); } const trimmedTitles = new Set(); @@ -319,10 +320,11 @@ export class Planner extends PlannerBase implements Agent { if (trimmedTitles.has(heading)) return section.text(); const count = section.query('blockquote').count(); if (count <= 10) return section.text(); - const kept = mdq(section.text()).query('blockquote[10:]').replace(''); + const kept = mdq(section.text()).query('blockquote[10:]').replace('').toString(); trimmedTitles.add(heading); return `${kept.trimEnd()}\n> ... and ${count - 10} more discoveries\n`; - }); + }) + .toString(); } return result.trim() || null; @@ -404,7 +406,7 @@ export class Planner extends PlannerBase implements Agent { if (this.scout && this.docsWeight > 0) { docsPromise = this.scout.collectDocs({ url: state.url, title: state.title, feature, excludeUrls: this.knowledgeTracker.applicationSpecUrls(state) }); } - let plannerResearch = mdq(research).query('code').replace(''); + let plannerResearch = mdq(research).query('code').replace('').toString(); plannerResearch = mdq(plannerResearch) .query('table') .replaceEach((table) => { @@ -415,7 +417,8 @@ export class Planner extends PlannerBase implements Agent { Type: r.Type || '', })); return jsonToTable(elementWithType, ['Element', 'Type']); - }); + }) + .toString(); const hasFocusedOverlay = hasFocusedSection(plannerResearch); const focusNote = hasFocusedOverlay ? "IMPORTANT: One section is marked as **Focused** — this is the user's current focus area. Concentrate testing on the Focused section FIRST — test all interactions inside it before planning tests for the rest of the page." : ''; diff --git a/src/ai/researcher.ts b/src/ai/researcher.ts index 358c48ed..09e08c61 100644 --- a/src/ai/researcher.ts +++ b/src/ai/researcher.ts @@ -313,7 +313,7 @@ export class Researcher extends ResearcherBase implements Agent { researchFile = saveResearch(researchState, result.text, combinedHtml); } - const summaryText = mdq(result.text).query('section2(/^summary/)').query('paragraph[0]').text().trim(); + const summaryText = mdq(result.text).query('section2(/^summary/i)').query('paragraph[0]').text().trim(); const summaryLine = summaryText.split('\n')[0]?.trim().slice(0, 200); if (summaryLine) this.experienceTracker.updateSummary(this.actionResult!, summaryLine); diff --git a/src/ai/researcher/deep-analysis.ts b/src/ai/researcher/deep-analysis.ts index a5414167..16d10a6d 100644 --- a/src/ai/researcher/deep-analysis.ts +++ b/src/ai/researcher/deep-analysis.ts @@ -126,7 +126,7 @@ export function WithDeepAnalysis(Base: T) { let updated: string; if (extQuery.count() > 0) { const existing = extQuery.text().trimEnd(); - updated = extQuery.replace(`${existing}\n\n${sectionMarkdown}\n`); + updated = extQuery.replace(`${existing}\n\n${sectionMarkdown}\n`).toString(); } else { updated = `${cached.trimEnd()}\n\n# Extended Research\n\n${sectionMarkdown}\n`; } @@ -539,7 +539,7 @@ export function WithDeepAnalysis(Base: T) { if (heading.count() === 0) heading = mdq(sectionMarkdown).query('h2[0]'); if (heading.count() === 0) return sectionMarkdown; - return heading.replace(`${heading.text().trimEnd()}\n\n> Container: '${containerCss}'\n\n`); + return heading.replace(`${heading.text().trimEnd()}\n\n> Container: '${containerCss}'\n\n`).toString(); } private _deduplicateExpandedSections(sections: string[]): string[] { diff --git a/src/ai/researcher/locators.ts b/src/ai/researcher/locators.ts index cfe0d278..e4746258 100644 --- a/src/ai/researcher/locators.ts +++ b/src/ai/researcher/locators.ts @@ -304,9 +304,9 @@ export function WithLocators(Base: T) { if (sectionQuery.count() === 0) sectionQuery = mdq(result.text).query(`section3(~"${escaped}")`); if (newCss) { - result.text = sectionQuery.query('blockquote[0]').setKeyValue('Container', `'${newCss}'`); + result.text = sectionQuery.query('blockquote[0]').setKeyValue('Container', `'${newCss}'`).toString(); } else { - result.text = sectionQuery.query('blockquote[0]').replace(''); + result.text = sectionQuery.query('blockquote[0]').replace('').toString(); result.text = result.text.replace(`${FOCUSED_MARKER}\n`, ''); } diff --git a/src/ai/researcher/pagination.ts b/src/ai/researcher/pagination.ts index af24a204..8e405ac0 100644 --- a/src/ai/researcher/pagination.ts +++ b/src/ai/researcher/pagination.ts @@ -58,7 +58,7 @@ export function WithPagination(Base: T) { let sectionQuery = mdq(result.text).query(`section2(~"${escaped}")`); if (sectionQuery.count() === 0) sectionQuery = mdq(result.text).query(`section3(~"${escaped}")`); if (sectionQuery.count() === 0) return; - result.text = sectionQuery.query('blockquote[0]').setKeyValue('Pagination', strategy); + result.text = sectionQuery.query('blockquote[0]').setKeyValue('Pagination', strategy).toString(); } }; } diff --git a/src/ai/researcher/research-result.ts b/src/ai/researcher/research-result.ts index 7eeece7b..a2cbbea0 100644 --- a/src/ai/researcher/research-result.ts +++ b/src/ai/researcher/research-result.ts @@ -52,9 +52,9 @@ export class ResearchResult { const escaped = section.name.replace(/"/g, '\\"'); let sectionQuery = mdq(this.text).query(`section2(~"${escaped}")`); if (sectionQuery.count() === 0) sectionQuery = mdq(this.text).query(`section3(~"${escaped}")`); - const updated = sectionQuery.query('table').replace(`${newTable.trimEnd()}\n`); + const updated = sectionQuery.query('table').replace(`${newTable.trimEnd()}\n`).toString(); if (updated === this.text) return; - section.rawMarkdown = mdq(section.rawMarkdown).query('table').replace(`${newTable.trimEnd()}\n`); + section.rawMarkdown = mdq(section.rawMarkdown).query('table').replace(`${newTable.trimEnd()}\n`).toString(); this.text = updated; } diff --git a/src/experience-tracker.ts b/src/experience-tracker.ts index 12690f0b..89b0368d 100644 --- a/src/experience-tracker.ts +++ b/src/experience-tracker.ts @@ -262,7 +262,7 @@ export class ExperienceTracker { while (content.split('\n').length > maxLines) { const sections = mdq(content).query('section2').each(); if (sections.length <= 1) break; - content = sections[sections.length - 1].replace(''); + content = sections[sections.length - 1].replace('').toString(); } return { ...experience, content }; }); @@ -286,7 +286,7 @@ export class ExperienceTracker { combined = renderAsHowTo(combined); if (options?.stripCode) { - combined = mdq(combined).query('code').replace(''); + combined = mdq(combined).query('code').replace('').toString(); } if (combined.trim()) results.push(combined.trim()); @@ -515,7 +515,8 @@ function renderAsHowTo(content: string): string { if (text.startsWith('FLOW:')) return `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`; if (text.startsWith('ACTION:')) return `## HOW to ${text.slice(7).trim()} (single-step)\n\n`; return heading.text(); - }); + }) + .toString(); } export interface ExperienceFile { diff --git a/src/knowledge-tracker.ts b/src/knowledge-tracker.ts index 24d05c9e..a1e5ada2 100644 --- a/src/knowledge-tracker.ts +++ b/src/knowledge-tracker.ts @@ -214,7 +214,7 @@ export class KnowledgeTracker { return this.knowledgeFiles.map((knowledge) => { const content = knowledge.content.trim(); - const firstLine = mdq(content).meta()[0]?.text.split('\n')[0]?.trim() || ''; + const firstLine = mdq(content).nodes()[0]?.text.split('\n')[0]?.trim() || ''; return { url: knowledge.url || knowledge.endpoint || '', firstLine, diff --git a/src/utils/markdown-query.ts b/src/utils/markdown-query.ts index 0930a27f..423d4635 100644 --- a/src/utils/markdown-query.ts +++ b/src/utils/markdown-query.ts @@ -1,519 +1 @@ -import { type Token, type Tokens, marked } from 'marked'; - -export type SelectorType = 'section' | 'section1' | 'section2' | 'section3' | 'section4' | 'section5' | 'section6' | 'table' | 'heading' | 'paragraph' | 'list' | 'item' | 'code' | 'blockquote' | 'hr' | 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6'; - -export interface TextMatcher { - mode: 'exact' | 'contains' | 'regex'; - value: string; - negated: boolean; -} - -export interface QuerySegment { - selector: SelectorType; - textMatch?: TextMatcher; - index: number | null; - slice: { from?: number; to?: number } | null; -} - -export interface MatchedRange { - token: Token; - start: number; - length: number; - innerTokens?: MatchedRange[]; -} - -export function parseQuery(input: string): QuerySegment[] { - const segments: QuerySegment[] = []; - let pos = 0; - - function peek(): string { - return pos < input.length ? input[pos] : ''; - } - - function advance(): string { - return input[pos++] || ''; - } - - function skipWhitespace() { - while (pos < input.length && /\s/.test(input[pos])) pos++; - } - - function readIdentifier(): string { - const start = pos; - while (pos < input.length && /[a-zA-Z_\d]/.test(input[pos])) pos++; - return input.slice(start, pos); - } - - function readQuotedString(): string { - const quote = advance(); - let result = ''; - while (pos < input.length && input[pos] !== quote) { - if (input[pos] === '\\') { - pos++; - result += input[pos] || ''; - } else { - result += input[pos]; - } - pos++; - } - if (pos < input.length) pos++; - return result; - } - - function readUntilAny(chars: string): string { - const start = pos; - while (pos < input.length && !chars.includes(input[pos])) pos++; - return input.slice(start, pos); - } - - function parseTextMatcher(): TextMatcher { - let negated = false; - if (peek() === '!') { - negated = true; - advance(); - } - - if (peek() === '~') { - advance(); - const value = readQuotedString(); - return { mode: 'contains', value, negated }; - } - - if (peek() === '/') { - advance(); - let value = ''; - while (pos < input.length && input[pos] !== '/') { - value += input[pos]; - pos++; - } - if (pos < input.length) pos++; - while (pos < input.length && /[gimsuy]/.test(input[pos])) pos++; - return { mode: 'regex', value, negated }; - } - - const value = readQuotedString(); - return { mode: 'exact', value, negated }; - } - - while (pos < input.length) { - skipWhitespace(); - if (pos >= input.length) break; - - const selector = readIdentifier(); - if (!selector) { - pos++; - continue; - } - - const segment: QuerySegment = { - selector: selector as SelectorType, - index: null, - slice: null, - }; - - if (peek() === '(') { - advance(); - segment.textMatch = parseTextMatcher(); - if (peek() === ')') advance(); - } - - while (peek() === '[') { - advance(); - const content = readUntilAny(']'); - if (/^-?\d*(:-?\d*)?$/.test(content) && content !== '') { - if (content.includes(':')) { - const colonIdx = content.indexOf(':'); - const fromStr = content.slice(0, colonIdx); - const toStr = content.slice(colonIdx + 1); - segment.slice = { - from: fromStr ? Number.parseInt(fromStr, 10) : undefined, - to: toStr ? Number.parseInt(toStr, 10) : undefined, - }; - } else { - segment.index = Number.parseInt(content, 10); - } - } - if (peek() === ']') advance(); - } - - segments.push(segment); - } - - return segments; -} - -function matchText(text: string, matcher: TextMatcher): boolean { - let result: boolean; - - switch (matcher.mode) { - case 'exact': - result = text === matcher.value; - break; - case 'contains': - result = text.includes(matcher.value); - break; - case 'regex': - result = new RegExp(matcher.value, 'i').test(text); - break; - default: - result = false; - } - - return matcher.negated ? !result : result; -} - -function entryKey(line: string): string | null { - const separator = line.indexOf(':'); - if (separator < 1) return null; - return line.slice(0, separator).trim().toLowerCase(); -} - -function getTokenText(token: Token): string { - const t = token as any; - switch (token.type) { - case 'heading': - case 'paragraph': - case 'code': - case 'blockquote': - case 'list_item': - return t.text || ''; - case 'table': - return (t.header || []).map((h: any) => h.text).join(', '); - default: - return ''; - } -} - -function getHeadingDepth(selector: string): number | null { - const match = selector.match(/^h([1-6])$/); - return match ? Number.parseInt(match[1], 10) : null; -} - -function isSectionSelector(selector: string): boolean { - return /^section\d?$/.test(selector); -} - -function getSectionDepth(selector: string): number | null { - const match = selector.match(/^section([1-6])$/); - return match ? Number.parseInt(match[1], 10) : null; -} - -function selectorToTokenType(selector: string): string | null { - if (/^h[1-6]$/.test(selector)) return 'heading'; - const map: Record = { - heading: 'heading', - paragraph: 'paragraph', - table: 'table', - code: 'code', - list: 'list', - blockquote: 'blockquote', - hr: 'hr', - item: 'list_item', - }; - return map[selector] || null; -} - -function buildTokenIndex(source: string): MatchedRange[] { - const tokens = marked.lexer(source); - const ranges: MatchedRange[] = []; - let offset = 0; - - for (const token of tokens) { - const raw = (token as any).raw || ''; - ranges.push({ token, start: offset, length: raw.length }); - offset += raw.length; - } - - return ranges; -} - -function computeSections(candidates: MatchedRange[], segment: QuerySegment): MatchedRange[] { - const sectionDepth = getSectionDepth(segment.selector as string); - const sections: MatchedRange[] = []; - - for (let i = 0; i < candidates.length; i++) { - const range = candidates[i]; - if (range.token.type !== 'heading') continue; - - const heading = range.token as Tokens.Heading; - if (sectionDepth !== null && heading.depth !== sectionDepth) continue; - if (segment.textMatch && !matchText(heading.text, segment.textMatch)) continue; - - const depth = heading.depth; - const innerTokens: MatchedRange[] = []; - let endOffset = range.start + range.length; - - for (let j = i + 1; j < candidates.length; j++) { - const nextRange = candidates[j]; - if (nextRange.token.type === 'heading' && (nextRange.token as Tokens.Heading).depth <= depth) break; - innerTokens.push(nextRange); - endOffset = nextRange.start + nextRange.length; - } - - sections.push({ - token: range.token, - start: range.start, - length: endOffset - range.start, - innerTokens, - }); - } - - return sections; -} - -function extractListItems(candidates: MatchedRange[]): MatchedRange[] { - const items: MatchedRange[] = []; - - for (const range of candidates) { - if (range.token.type !== 'list') continue; - - const list = range.token as Tokens.List; - const listRaw = (range.token as any).raw as string; - let searchFrom = 0; - - for (const item of list.items) { - const itemRaw = (item as any).raw as string; - const idx = listRaw.indexOf(itemRaw, searchFrom); - if (idx === -1) continue; - - items.push({ - token: item as unknown as Token, - start: range.start + idx, - length: itemRaw.length, - }); - - searchFrom = idx + itemRaw.length; - } - } - - return items; -} - -function applyIndexSlice(matches: MatchedRange[], segment: QuerySegment): MatchedRange[] { - if (segment.index !== null) { - const idx = segment.index < 0 ? matches.length + segment.index : segment.index; - return idx >= 0 && idx < matches.length ? [matches[idx]] : []; - } - - if (segment.slice) { - const { from, to } = segment.slice; - return matches.slice(from, to); - } - - return matches; -} - -function expandSectionRanges(matches: MatchedRange[]): MatchedRange[] { - let hasSection = false; - for (const m of matches) { - if (m.innerTokens) { - hasSection = true; - break; - } - } - if (!hasSection) return matches; - - const expanded: MatchedRange[] = []; - for (const m of matches) { - if (m.innerTokens) { - expanded.push({ token: m.token, start: m.start, length: ((m.token as any).raw || '').length }); - expanded.push(...m.innerTokens); - } else { - expanded.push(m); - } - } - return expanded; -} - -function executeSegments(candidates: MatchedRange[], segments: QuerySegment[]): MatchedRange[] { - if (segments.length === 0) return candidates; - - const segment = segments[0]; - const remaining = segments.slice(1); - - if (isSectionSelector(segment.selector as string)) { - const sections = computeSections(candidates, segment); - const indexed = applyIndexSlice(sections, segment); - - if (remaining.length === 0) return indexed; - - const results: MatchedRange[] = []; - for (const section of indexed) { - results.push(...executeSegments(section.innerTokens || [], remaining)); - } - return results; - } - - if (segment.selector === 'item') { - let items = extractListItems(candidates); - if (segment.textMatch) { - items = items.filter((r) => matchText(getTokenText(r.token), segment.textMatch!)); - } - return executeSegments(applyIndexSlice(items, segment), remaining); - } - - const tokenType = selectorToTokenType(segment.selector as string); - if (!tokenType) return []; - - let matches = candidates.filter((r) => r.token.type === tokenType); - - const depth = getHeadingDepth(segment.selector as string); - if (depth !== null) { - matches = matches.filter((r) => (r.token as any).depth === depth); - } - - if (segment.textMatch) { - matches = matches.filter((r) => matchText(getTokenText(r.token), segment.textMatch!)); - } - - return executeSegments(applyIndexSlice(matches, segment), remaining); -} - -export class MarkdownQuery { - private source: string; - private matches: MatchedRange[]; - - constructor(source: string, matches?: MatchedRange[]) { - this.source = source; - this.matches = matches || buildTokenIndex(source); - } - - query(selector: string): MarkdownQuery { - const segments = parseQuery(selector); - const candidates = expandSectionRanges(this.matches); - const results = executeSegments(candidates, segments); - return new MarkdownQuery(this.source, results); - } - - text(): string { - return this.matches.map((r) => this.source.slice(r.start, r.start + r.length)).join(''); - } - - get(): string { - return this.text(); - } - - toJson(): Record[] { - const results: Record[] = []; - - for (const range of this.matches) { - if (range.token.type !== 'table') continue; - - const table = range.token as Tokens.Table; - const headers = table.header.map((h) => h.text); - for (const row of table.rows) { - const obj: Record = {}; - for (let i = 0; i < headers.length; i++) { - obj[headers[i]] = row[i]?.text ?? ''; - } - results.push(obj); - } - } - - return results; - } - - keyValue(): Record { - const entries: Record = {}; - - for (const range of this.matches) { - for (const line of getTokenText(range.token).split('\n')) { - const key = entryKey(line); - if (!key) continue; - const value = line.slice(line.indexOf(':') + 1).trim(); - if (value) entries[key] = value; - } - } - - return entries; - } - - setKeyValue(key: string, value: string | null): string { - return this.replaceEach((match) => { - const token = match.matches[0].token; - const lines = getTokenText(token) - .split('\n') - .map((line) => line.trim()) - .filter(Boolean); - - const index = lines.findIndex((line) => entryKey(line) === key.toLowerCase()); - if (index < 0 && value) lines.push(`${key}: ${value}`); - if (index >= 0 && value) lines[index] = `${key}: ${value}`; - if (index >= 0 && !value) lines.splice(index, 1); - - if (token.type !== 'blockquote') return lines.join('\n'); - return lines.map((line) => `> ${line}`).join('\n'); - }); - } - - replace(content: string): string { - return this.replaceEach(() => content); - } - - replaceEach(replacer: (match: MarkdownQuery, index: number) => string): string { - const sorted = [...this.matches].sort((a, b) => a.start - b.start); - - const kept: MatchedRange[] = []; - let lastEnd = -1; - for (const range of sorted) { - if (range.start < lastEnd) continue; - kept.push(range); - lastEnd = range.start + range.length; - } - - const replacements = kept.map((range, index) => replacer(new MarkdownQuery(this.source, [range]), index)); - let result = this.source; - for (let i = kept.length - 1; i >= 0; i--) { - const range = kept[i]; - result = result.slice(0, range.start) + replacements[i] + result.slice(range.start + range.length); - } - - return result; - } - - count(): number { - return this.matches.length; - } - - first(): MarkdownQuery { - return new MarkdownQuery(this.source, this.matches.slice(0, 1)); - } - - last(): MarkdownQuery { - return new MarkdownQuery(this.source, this.matches.slice(-1)); - } - - before(): MarkdownQuery { - if (this.matches.length === 0) return new MarkdownQuery(this.source, []); - const cutoff = this.matches[0].start; - const allTokens = buildTokenIndex(this.source); - const beforeTokens = allTokens.filter((r) => r.start + r.length <= cutoff); - return new MarkdownQuery(this.source, beforeTokens); - } - - after(): MarkdownQuery { - if (this.matches.length === 0) return new MarkdownQuery(this.source, []); - const lastMatch = this.matches[this.matches.length - 1]; - const cutoff = lastMatch.start + lastMatch.length; - const allTokens = buildTokenIndex(this.source); - const afterTokens = allTokens.filter((r) => r.start >= cutoff); - return new MarkdownQuery(this.source, afterTokens); - } - - each(): MarkdownQuery[] { - return this.matches.map((m) => new MarkdownQuery(this.source, [m])); - } - - meta(): Array<{ type: string; depth: number | null; text: string }> { - return this.matches.map((range) => { - const token = range.token as any; - let depth: number | null = null; - if (token.type === 'heading') depth = token.depth; - return { type: token.type, depth, text: getTokenText(range.token) }; - }); - } -} - -export function mdq(source: string): MarkdownQuery { - return new MarkdownQuery(source); -} +export * from './mdq/query.ts'; diff --git a/src/utils/mdq/README.md b/src/utils/mdq/README.md new file mode 100644 index 00000000..8b2bdd8a --- /dev/null +++ b/src/utils/mdq/README.md @@ -0,0 +1,187 @@ +# mdq + +Query and edit markdown with a selector language — jq, for markdown. + +## Install + +```bash +npx mdq 'h2' README.md # no install +npm install mdq # as a library +npm install -g mdq # as a command +``` + +Node 18 or newer. Two dependencies: `marked` and `yaml`. + +## Use + +```js +import { mdq } from 'mdq'; + +mdq(readme).query('section("Install") code[0]').text(); +mdq(plan).comment(/^test/).nodes(); +mdq(doc).table().addRow({ Method: 'POST', Path: '/sessions' }).toString(); +``` + +There is a CLI too: + +```bash +mdq 'section("API") table' --json README.md +``` + +## The one rule + +**Reads narrow, writes return the document.** + +`mdq(source)` gives a `MarkdownQuery` — one class, holding the document and the set of +blocks currently selected. `query()` and the sugar methods narrow that set. Every write +returns a fresh `MarkdownQuery` over the edited document, so edits chain and end with +`toString()`: + +```js +mdq(source) + .query('section("API")').append('## Notes\n') + .query('blockquote[0]').remove() + .toString(); +``` + +`toString()` is always the whole document; `text()` is the markdown of the current +selection. + +## Selectors + +| Selector | Matches | +| --- | --- | +| `section` | a heading and everything under it, until the next heading of the same or shallower depth | +| `section1` … `section6` | the same, restricted to one heading depth | +| `heading`, `h1` … `h6` | the heading line alone | +| `paragraph` | a paragraph | +| `table` | a GFM table | +| `list` | a bullet or ordered list | +| `item` | one item of a list | +| `code` | a fenced code block | +| `blockquote` | a `>` block | +| `hr` | a thematic break | +| `html` | an HTML block, matched on its raw text | +| `comment` | an HTML comment, matched on its **inner** body | + +Text matchers go in parentheses, and `!` negates any of them: + +``` +section("Install") exact +section(~"Inst") contains +section(/^inst/i) regex, with its own flags +section(!~"Draft") negated +``` + +Index and slice with brackets, and compose with spaces to scope one selector inside +another: + +``` +heading[0] first +heading[-1] last +blockquote[2:5] a slice +section("API") table every table inside that section +``` + +A leading `.` is accepted and ignored, so `.h2` works if that is your habit from jq. + +An unknown selector throws `MdqSelectorError`, which carries the `index` of the offending +character. It never silently matches nothing. + +## Matchers as values + +Passing a JavaScript value avoids escaping a dynamic string into a selector: + +```js +mdq(doc).query('section2', section.name); // exact +mdq(doc).heading(/^summary/i); // regex, own flags +mdq(doc).item((text) => text.length > 80); // predicate +``` + +A `string` matches exactly, a `RegExp` honors its own flags, and a function is a predicate +over the node's text. Every sugar method takes one: `section` `heading` `paragraph` `table` +`list` `item` `code` `blockquote` `comment` `html` `hr`. Each is exactly +`query(selector, matcher)`; `section` and `heading` also take `{ depth }`. + +## Reading + +| Method | Returns | +| --- | --- | +| `text()` | raw markdown of every match, joined | +| `nodes()` | `{ type, depth, text }` per match | +| `rows()` | table rows as objects, keyed by header | +| `entries()` | `Key: value` lines of a block, keys lowercased | +| `count()` / `exists()` | how many matched / whether any did | +| `first()` / `last()` / `at(n)` / `slice(from, to)` | narrow the selection | +| `each()` | one single-match query per match | +| `preceding()` / `following()` | everything before the first / after the last match | + +## Writing + +Every one returns a `MarkdownQuery` over the edited document. + +| Method | Effect | +| --- | --- | +| `replace(md)` | replace each match | +| `replaceEach(fn)` | replace each match with `fn(selection, index)` | +| `remove()` | delete each match, and its blank line | +| `insertBefore(md)` / `insertAfter(md)` | add a sibling block | +| `prepend(md)` / `append(md)` | add a block inside a section or list | +| `addRow(obj)` | append a table row, re-aligning the columns | +| `addItem(text)` | append a list item, copying the existing marker | +| `setEntry(key, value)` | set a `Key: value` line; `null` deletes it | + +`prepend` and `append` need a section or list; on any other block they raise +`MdqOperationError`. Anything that takes markdown also takes a `MarkdownQuery`. Writes +never leave zero blank lines between blocks, and never more than one — including inside +fenced code blocks, which are left exactly as they are. + +## Frontmatter + +A leading `---` block is parsed as YAML, kept out of the token index, and exposed as data. +Without this, `marked` reads `url: /login` as a setext heading. + +```js +const doc = mdq(page); +doc.frontmatter(); // { url: '/login', wait: 1000, tags: ['auth'] } +doc.query('h2').count(); // 0 — the --- block is not a heading +doc.setFrontmatter('wait', 2000); // comments and formatting survive +``` + +## Prior art + +The selector grammar here is **bespoke**. It is not a standard, and there is no upstream +parser for it — element names and descendant-by-space come from CSS, `[2:5]` slices from +Python, and a tolerated leading `.` from jq. + +The established alternative is the **unified/remark** stack: parse to +[mdast](https://github.com/syntax-tree/mdast), then select with +[`unist-util-select`](https://github.com/syntax-tree/unist-util-select), which implements +real CSS selectors on top of `css-selector-parser`. If you want a standards-based tool, +use that. + +mdq exists because three things do not fall out of that stack: + +- **Sections.** mdast is flat: a heading and the blocks beneath it are siblings, so the CSS + descendant combinator cannot express "this heading and everything under it until the next + heading of the same depth". [`remark-sectionize`](https://github.com/jake-low/remark-sectionize) + adds the nesting, but its synthetic `section` nodes carry no `position`, so their source + range has to be derived from their children before anything can be edited in place. +- **Editing by byte range.** mdq records each block's offset in the original source and + splices text, so anything it does not touch stays byte-identical. mdast nodes do carry + offsets, so this is achievable there too — it is a thing to build, not a thing you get. +- **Text and pattern matching.** mdast headings have no flat text field (the text is a child + node), CSS dropped `:contains()`, and `unist-util-select` parses the attribute `i` flag + but does not apply it — so `/^summary/i` has no selector form at all. + +GFM tables are not in core remark either; they need `remark-gfm`. + + +## Limitations + +- **Block-level comments only.** A comment inside a paragraph (`text more`) is + part of that paragraph's token and is not reachable as a `comment`. +- **No row or item selectors.** `addRow` and `addItem` append; there is no `removeRow`, + because there is nothing to select. +- **YAML frontmatter only.** TOML (`+++`) and JSON blocks are skipped from the token index + but not parsed. diff --git a/src/utils/mdq/cli.ts b/src/utils/mdq/cli.ts new file mode 100644 index 00000000..9917d583 --- /dev/null +++ b/src/utils/mdq/cli.ts @@ -0,0 +1,118 @@ +import { readFileSync, writeFileSync } from 'node:fs'; +import { Command } from 'commander'; +import { type MarkdownQuery, MdqError, mdq } from './query.ts'; + +const EDIT_FLAGS = ['remove', 'replace', 'insertBefore', 'insertAfter', 'prepend', 'append', 'addRow', 'addItem', 'set'] as const; + +export async function runMdq(argv: string[], readStdin: StdinReader): Promise { + const program = new Command(); + program + .name('mdq') + .description('query and edit markdown') + .argument('[selector]', 'markdown selector') + .argument('[file]', 'file to read; stdin when omitted') + .option('-j, --json', 'output rows as JSON') + .option('-c, --count', 'print the number of matches') + .option('-t, --text', 'print unwrapped text') + .option('--frontmatter', 'print frontmatter as JSON') + .option('-i, --in-place', 'write the result back to the file') + .option('--remove', 'delete matched blocks') + .option('--replace ', 'replace matched blocks') + .option('--insert-before ', 'insert before each match') + .option('--insert-after ', 'insert after each match') + .option('--prepend ', 'insert at the start of each match') + .option('--append ', 'insert at the end of each match') + .option('--add-row ', 'append a table row') + .option('--add-item ', 'append a list item') + .option('--set ', 'set an entry; omit the value to delete it') + .exitOverride() + .configureOutput({ writeOut: () => {}, writeErr: () => {} }); + + try { + program.parse(argv, { from: 'user' }); + } catch (error) { + return { output: String((error as Error).message), code: 2 }; + } + + const options = program.opts(); + let [selector, file] = program.args; + if (options.frontmatter && selector && !file) { + file = selector; + selector = ''; + } + + let source = ''; + if (!file) source = await readStdin(); + if (file) { + try { + source = readFileSync(file, 'utf8'); + } catch { + return { output: `Cannot read ${file}`, code: 2 }; + } + } + + try { + return await apply(mdq(source), selector, options, file); + } catch (error) { + if (error instanceof MdqError) return { output: error.message, code: 2 }; + throw error; + } +} + +async function apply(doc: MarkdownQuery, selector: string, options: Record, file?: string): Promise { + if (options.frontmatter) return { output: `${JSON.stringify(doc.frontmatter(), null, 2)}\n`, code: 0 }; + if (!selector) return { output: 'A selector is required', code: 2 }; + + const chosen = EDIT_FLAGS.filter((flag) => options[flag] !== undefined); + if (chosen.length > 1) return { output: 'Only one edit at a time', code: 2 }; + + const selection = doc.query(selector); + if (chosen.length === 0) return read(selection, options); + + if (!selection.exists()) return finish(String(doc), 1, options, file); + return finish(String(edit(selection, chosen[0], options)), 0, options, file); +} + +function read(selection: MarkdownQuery, options: Record): CliResult { + let code = 1; + if (selection.exists()) code = 0; + if (options.count) return { output: `${selection.count()}\n`, code: 0 }; + if (options.json) return { output: `${JSON.stringify(selection.rows(), null, 2)}\n`, code }; + if (options.text) + return { + output: selection + .nodes() + .map((node) => node.text) + .join('\n\n'), + code, + }; + return { output: selection.text(), code }; +} + +function edit(selection: MarkdownQuery, flag: string, options: Record): MarkdownQuery { + if (flag === 'remove') return selection.remove(); + if (flag === 'replace') return selection.replace(options.replace); + if (flag === 'insertBefore') return selection.insertBefore(options.insertBefore); + if (flag === 'insertAfter') return selection.insertAfter(options.insertAfter); + if (flag === 'prepend') return selection.prepend(options.prepend); + if (flag === 'append') return selection.append(options.append); + if (flag === 'addItem') return selection.addItem(options.addItem); + if (flag === 'addRow') return selection.addRow(JSON.parse(options.addRow)); + const separator = options.set.indexOf('='); + if (separator < 0) return selection.setEntry(options.set, null); + return selection.setEntry(options.set.slice(0, separator), options.set.slice(separator + 1) || null); +} + +function finish(output: string, code: number, options: Record, file?: string): CliResult { + if (!options.inPlace) return { output, code }; + if (!file) return { output: '--in-place needs a file', code: 2 }; + writeFileSync(file, output); + return { output: '', code }; +} + +export type StdinReader = () => Promise; + +export interface CliResult { + output: string; + code: number; +} diff --git a/src/utils/mdq/edit.ts b/src/utils/mdq/edit.ts new file mode 100644 index 00000000..152709a4 --- /dev/null +++ b/src/utils/mdq/edit.ts @@ -0,0 +1,158 @@ +import YAML from 'yaml'; +import type { MatchedRange } from './query.ts'; + +export class MarkdownEditor { + private source: string; + + constructor(source: string) { + this.source = source; + } + + static splitFrontmatter(source: string): FrontmatterSplit { + if (!source.startsWith('---')) return { raw: '', body: source, offset: 0 }; + const match = source.match(/^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/); + if (!match) return { raw: '', body: source, offset: 0 }; + return { raw: match[1], body: source.slice(match[0].length), offset: match[0].length }; + } + + static blockEnd(range: MatchedRange): number { + if (range.trailing) return range.trailing.start + range.trailing.length; + return range.start + range.length; + } + + static table(headers: string[], rows: string[][], align: (string | null)[]): string { + const widths = headers.map((header, index) => Math.max(header.length, 3, ...rows.map((row) => (row[index] || '').length))); + const line = (cells: string[]) => `| ${cells.map((cell, index) => (cell || '').padEnd(widths[index])).join(' | ')} |`; + const divider = `| ${widths.map((width, index) => MarkdownEditor.dashes(align[index], width)).join(' | ')} |`; + return `${[line(headers), divider, ...rows.map(line)].join('\n')}\n`; + } + + static item(listRaw: string, text: string): string { + const lines = listRaw.split('\n').filter((line) => line.trim()); + const last = lines[lines.length - 1] || '- x'; + const ordered = last.match(/^(\s*)(\d+)([.)])\s/); + if (ordered) return `${ordered[1]}${Number.parseInt(ordered[2], 10) + 1}${ordered[3]} ${text}`; + const bullet = last.match(/^(\s*)([-*+])\s/); + if (!bullet) return `- ${text}`; + return `${bullet[1]}${bullet[2]} ${text}`; + } + + static entries(text: string, key: string, value: string | null, isBlockquote: boolean): string { + const lines = text + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); + + const index = lines.findIndex((line) => MarkdownEditor.entryKey(line) === key.toLowerCase()); + if (index < 0 && value) lines.push(`${key}: ${value}`); + if (index >= 0 && value) lines[index] = `${key}: ${value}`; + if (index >= 0 && !value) lines.splice(index, 1); + + if (!isBlockquote) return lines.join('\n'); + return lines.map((line) => `> ${line}`).join('\n'); + } + + remove(ranges: MatchedRange[]): string { + const ordered = this.dedupe(ranges); + let result = this.source; + + for (let i = ordered.length - 1; i >= 0; i--) { + const range = ordered[i]; + const head = result.slice(0, range.start); + const tail = result.slice(MarkdownEditor.blockEnd(range)); + if (tail) { + result = head + tail; + continue; + } + if (!head) { + result = ''; + continue; + } + result = `${head.trimEnd()}\n`; + } + + return result; + } + + replace(ranges: MatchedRange[], render: (range: MatchedRange, index: number) => string): string { + const ordered = this.dedupe(ranges); + const rendered = ordered.map(render); + let result = this.source; + + for (let i = ordered.length - 1; i >= 0; i--) { + const range = ordered[i]; + result = result.slice(0, range.start) + rendered[i] + result.slice(range.start + range.length); + } + + return result; + } + + insert(offsets: number[], markdown: string): string { + const ordered = [...offsets].sort((a, b) => a - b); + let result = this.source; + for (let i = ordered.length - 1; i >= 0; i--) { + result = MarkdownEditor.spliceIn(result, ordered[i], markdown); + } + return result; + } + + frontmatter(): Record { + const { raw } = MarkdownEditor.splitFrontmatter(this.source); + if (!raw) return {}; + return (YAML.parseDocument(raw).toJS() as Record) || {}; + } + + setFrontmatter(key: string, value: unknown): string { + const { raw, body, offset } = MarkdownEditor.splitFrontmatter(this.source); + let document = new YAML.Document({}); + if (raw) document = YAML.parseDocument(raw); + if (value === null) document.delete(key); + if (value !== null) document.set(key, value); + + const rendered = document.toString().trimEnd(); + if (!offset) return `---\n${rendered}\n---\n\n${this.source}`; + return `---\n${rendered}\n---\n${body}`; + } + + private static spliceIn(source: string, offset: number, markdown: string): string { + const block = `${markdown.trimEnd()}\n`; + const before = source.slice(0, offset).replace(/\n+$/, ''); + const after = source.slice(offset).replace(/^\n+/, ''); + if (!before) return `${block}\n${after}`; + if (!after) return `${before}\n\n${block}`; + return `${before}\n\n${block}\n${after}`; + } + + private static dashes(alignment: string | null, width: number): string { + if (alignment === 'center') return `:${'-'.repeat(Math.max(width - 2, 1))}:`; + if (alignment === 'left') return `:${'-'.repeat(Math.max(width - 1, 1))}`; + if (alignment === 'right') return `${'-'.repeat(Math.max(width - 1, 1))}:`; + return '-'.repeat(width); + } + + private static entryKey(line: string): string | null { + const separator = line.indexOf(':'); + if (separator < 1) return null; + return line.slice(0, separator).trim().toLowerCase(); + } + + private dedupe(ranges: MatchedRange[]): MatchedRange[] { + const sorted = [...ranges].sort((a, b) => a.start - b.start); + const kept: MatchedRange[] = []; + let lastEnd = -1; + + for (const range of sorted) { + if (range.start < lastEnd) continue; + kept.push(range); + lastEnd = range.start + range.length; + } + + return kept; + } +} + +export interface FrontmatterSplit { + raw: string; + body: string; + offset: number; +} diff --git a/src/utils/mdq/package.json b/src/utils/mdq/package.json new file mode 100644 index 00000000..a048ac85 --- /dev/null +++ b/src/utils/mdq/package.json @@ -0,0 +1,28 @@ +{ + "name": "mdq", + "version": "0.0.0", + "description": "Query and edit markdown with a selector language - jq, for markdown", + "license": "MIT", + "type": "module", + "main": "./index.js", + "types": "./types/query.d.ts", + "exports": { + ".": { + "types": "./types/query.d.ts", + "import": "./index.js" + } + }, + "bin": { + "mdq": "./bin/mdq.js" + }, + "files": ["index.js", "types/", "bin/", "README.md"], + "engines": { + "node": ">=18" + }, + "keywords": ["markdown", "query", "selector", "cli", "jq", "frontmatter", "marked", "edit"], + "repository": { + "type": "git", + "url": "https://github.com/testomatio/explorbot", + "directory": "src/utils/mdq" + } +} diff --git a/src/utils/mdq/query.ts b/src/utils/mdq/query.ts new file mode 100644 index 00000000..823f0380 --- /dev/null +++ b/src/utils/mdq/query.ts @@ -0,0 +1,556 @@ +import { type Token, type Tokens, marked } from 'marked'; +import { MarkdownEditor } from './edit.ts'; + +export { MarkdownEditor }; + +export class MarkdownQuery { + private source: string; + private matches: MatchedRange[]; + + constructor(source: string, matches?: MatchedRange[]) { + this.source = source; + this.matches = matches || buildTokenIndex(source); + } + + query(selector: string, matcher?: Matcher): MarkdownQuery { + const segments = parseQuery(selector); + if (matcher !== undefined && segments.length > 0) segments[segments.length - 1].textMatch = this.valueMatcher(matcher); + return new MarkdownQuery(this.source, this.run(this.expandSections(this.matches), segments)); + } + + section(matcher?: Matcher, options?: SelectorOptions): MarkdownQuery { + return this.query(`section${options?.depth || ''}`, matcher); + } + + heading(matcher?: Matcher, options?: SelectorOptions): MarkdownQuery { + if (options?.depth) return this.query(`h${options.depth}`, matcher); + return this.query('heading', matcher); + } + + paragraph(matcher?: Matcher): MarkdownQuery { + return this.query('paragraph', matcher); + } + + table(matcher?: Matcher): MarkdownQuery { + return this.query('table', matcher); + } + + list(matcher?: Matcher): MarkdownQuery { + return this.query('list', matcher); + } + + item(matcher?: Matcher): MarkdownQuery { + return this.query('item', matcher); + } + + code(matcher?: Matcher): MarkdownQuery { + return this.query('code', matcher); + } + + blockquote(matcher?: Matcher): MarkdownQuery { + return this.query('blockquote', matcher); + } + + comment(matcher?: Matcher): MarkdownQuery { + return this.query('comment', matcher); + } + + html(matcher?: Matcher): MarkdownQuery { + return this.query('html', matcher); + } + + hr(): MarkdownQuery { + return this.query('hr'); + } + + text(): string { + return this.matches.map((range) => this.source.slice(range.start, range.start + range.length)).join(''); + } + + nodes(): NodeInfo[] { + return this.matches.map((range) => { + const token = range.token as any; + if (token.type !== 'heading') return { type: token.type, depth: null, text: this.tokenText(range.token) }; + return { type: token.type, depth: token.depth, text: this.tokenText(range.token) }; + }); + } + + rows(): Record[] { + const results: Record[] = []; + + for (const range of this.matches) { + if (range.token.type !== 'table') continue; + + const table = range.token as Tokens.Table; + const headers = table.header.map((cell) => cell.text); + for (const row of table.rows) { + const entry: Record = {}; + for (let i = 0; i < headers.length; i++) { + entry[headers[i]] = row[i]?.text ?? ''; + } + results.push(entry); + } + } + + return results; + } + + entries(): Record { + const entries: Record = {}; + + for (const range of this.matches) { + for (const line of this.tokenText(range.token).split('\n')) { + const separator = line.indexOf(':'); + if (separator < 1) continue; + const value = line.slice(separator + 1).trim(); + if (value) entries[line.slice(0, separator).trim().toLowerCase()] = value; + } + } + + return entries; + } + + frontmatter(): Record { + return this.editor().frontmatter(); + } + + count(): number { + return this.matches.length; + } + + exists(): boolean { + return this.matches.length > 0; + } + + at(index: number): MarkdownQuery { + let resolved = index; + if (resolved < 0) resolved = this.matches.length + resolved; + if (resolved < 0 || resolved >= this.matches.length) return new MarkdownQuery(this.source, []); + return new MarkdownQuery(this.source, [this.matches[resolved]]); + } + + slice(from?: number, to?: number): MarkdownQuery { + return new MarkdownQuery(this.source, this.matches.slice(from, to)); + } + + first(): MarkdownQuery { + return this.slice(0, 1); + } + + last(): MarkdownQuery { + return this.slice(-1); + } + + each(): MarkdownQuery[] { + return this.matches.map((range) => new MarkdownQuery(this.source, [range])); + } + + preceding(): MarkdownQuery { + if (this.matches.length === 0) return new MarkdownQuery(this.source, []); + const cutoff = this.matches[0].start; + return new MarkdownQuery( + this.source, + buildTokenIndex(this.source).filter((range) => range.start + range.length <= cutoff) + ); + } + + following(): MarkdownQuery { + if (this.matches.length === 0) return new MarkdownQuery(this.source, []); + const last = this.matches[this.matches.length - 1]; + const cutoff = last.start + last.length; + return new MarkdownQuery( + this.source, + buildTokenIndex(this.source).filter((range) => range.start >= cutoff) + ); + } + + replace(content: Markdown): MarkdownQuery { + return this.replaceEach(() => content); + } + + replaceEach(replacer: (match: MarkdownQuery, index: number) => Markdown): MarkdownQuery { + return new MarkdownQuery(this.editor().replace(this.matches, (range, index) => String(replacer(new MarkdownQuery(this.source, [range]), index)))); + } + + remove(): MarkdownQuery { + return new MarkdownQuery(this.editor().remove(this.matches)); + } + + insertBefore(markdown: Markdown): MarkdownQuery { + return this.insertAll((range) => range.start, markdown); + } + + insertAfter(markdown: Markdown): MarkdownQuery { + return this.insertAll((range) => MarkdownEditor.blockEnd(range), markdown); + } + + prepend(markdown: Markdown): MarkdownQuery { + return this.insertAll((range) => this.containerStart(range), markdown); + } + + append(markdown: Markdown): MarkdownQuery { + return this.insertAll((range) => this.containerEnd(range), markdown); + } + + addRow(row: Record): MarkdownQuery { + return new MarkdownQuery( + this.editor().replace(this.matches, (range) => { + if (range.token.type !== 'table') throw new MdqOperationError(`addRow needs a table, got ${range.token.type}`); + const table = range.token as Tokens.Table; + const headers = table.header.map((cell) => cell.text); + const existing = table.rows.map((cells) => headers.map((_, index) => cells[index]?.text || '')); + return MarkdownEditor.table(headers, [...existing, headers.map((header) => row[header] || '')], table.align); + }) + ); + } + + addItem(text: string): MarkdownQuery { + return new MarkdownQuery( + this.editor().replace(this.matches, (range) => { + if (range.token.type !== 'list') throw new MdqOperationError(`addItem needs a list, got ${range.token.type}`); + const raw = (((range.token as any).raw as string) || '').trimEnd(); + return `${raw}\n${MarkdownEditor.item(raw, text)}\n`; + }) + ); + } + + setEntry(key: string, value: string | null): MarkdownQuery { + return this.replaceEach((match) => { + const token = match.matches[0].token; + return MarkdownEditor.entries(this.tokenText(token), key, value, token.type === 'blockquote'); + }); + } + + setFrontmatter(key: string, value: unknown): MarkdownQuery { + return new MarkdownQuery(this.editor().setFrontmatter(key, value)); + } + + toString(): string { + return this.source; + } + + valueOf(): string { + return this.source; + } + + /** Compatibility aliases. Each delegates to the canonical name above; prefer those. */ + get(): string { + return this.text(); + } + + meta(): NodeInfo[] { + return this.nodes(); + } + + toJson(): Record[] { + return this.rows(); + } + + keyValue(): Record { + return this.entries(); + } + + setKeyValue(key: string, value: string | null): MarkdownQuery { + return this.setEntry(key, value); + } + + before(): MarkdownQuery { + return this.preceding(); + } + + after(): MarkdownQuery { + return this.following(); + } + + private editor(): MarkdownEditor { + return new MarkdownEditor(this.source); + } + + private run(candidates: MatchedRange[], segments: QuerySegment[]): MatchedRange[] { + if (segments.length === 0) return candidates; + + const segment = segments[0]; + const remaining = segments.slice(1); + + if (/^section[1-6]?$/.test(segment.selector)) { + const sections = this.narrow(this.sectionsOf(candidates, segment), segment); + if (remaining.length === 0) return sections; + + const results: MatchedRange[] = []; + for (const section of sections) results.push(...this.run(section.innerTokens || [], remaining)); + return results; + } + + if (segment.selector === 'comment') { + const comments = candidates.filter((range) => this.isComment(range.token)); + return this.run(this.narrow(this.filterByText(comments, segment), segment), remaining); + } + + if (segment.selector === 'item') { + return this.run(this.narrow(this.filterByText(this.itemsOf(candidates), segment), segment), remaining); + } + + const depth = segment.selector.match(/^h([1-6])$/); + let type = TOKEN_ALIASES[segment.selector] || segment.selector; + if (depth) type = 'heading'; + + let matches = candidates.filter((range) => range.token.type === type); + if (depth) matches = matches.filter((range) => (range.token as any).depth === Number.parseInt(depth[1], 10)); + + return this.run(this.narrow(this.filterByText(matches, segment), segment), remaining); + } + + private sectionsOf(candidates: MatchedRange[], segment: QuerySegment): MatchedRange[] { + const wanted = segment.selector.match(/^section([1-6])$/); + const sections: MatchedRange[] = []; + + for (let i = 0; i < candidates.length; i++) { + const range = candidates[i]; + if (range.token.type !== 'heading') continue; + + const heading = range.token as Tokens.Heading; + if (wanted && heading.depth !== Number.parseInt(wanted[1], 10)) continue; + if (segment.textMatch && !this.matchText(heading.text, segment.textMatch)) continue; + + const innerTokens: MatchedRange[] = []; + let end = range.start + range.length; + + for (let j = i + 1; j < candidates.length; j++) { + const next = candidates[j]; + if (next.token.type === 'heading' && (next.token as Tokens.Heading).depth <= heading.depth) break; + innerTokens.push(next); + end = MarkdownEditor.blockEnd(next); + } + + sections.push({ token: range.token, start: range.start, length: end - range.start, innerTokens }); + } + + return sections; + } + + private itemsOf(candidates: MatchedRange[]): MatchedRange[] { + const items: MatchedRange[] = []; + + for (const range of candidates) { + if (range.token.type !== 'list') continue; + + const raw = (range.token as any).raw as string; + let cursor = 0; + + for (const item of (range.token as Tokens.List).items) { + const itemRaw = (item as any).raw as string; + const at = raw.indexOf(itemRaw, cursor); + if (at === -1) continue; + items.push({ token: item as unknown as Token, start: range.start + at, length: itemRaw.length }); + cursor = at + itemRaw.length; + } + } + + return items; + } + + private expandSections(matches: MatchedRange[]): MatchedRange[] { + if (!matches.some((range) => range.innerTokens)) return matches; + + const expanded: MatchedRange[] = []; + for (const range of matches) { + if (!range.innerTokens) { + expanded.push(range); + continue; + } + expanded.push({ token: range.token, start: range.start, length: ((range.token as any).raw || '').length }); + expanded.push(...range.innerTokens); + } + return expanded; + } + + private narrow(matches: MatchedRange[], segment: QuerySegment): MatchedRange[] { + if (segment.index !== null) { + let index = segment.index; + if (index < 0) index = matches.length + index; + if (index < 0 || index >= matches.length) return []; + return [matches[index]]; + } + if (segment.slice) return matches.slice(segment.slice.from, segment.slice.to); + return matches; + } + + private filterByText(matches: MatchedRange[], segment: QuerySegment): MatchedRange[] { + if (!segment.textMatch) return matches; + return matches.filter((range) => this.matchText(this.tokenText(range.token), segment.textMatch!)); + } + + private matchText(text: string, matcher: TextMatcher): boolean { + let result = false; + if (matcher.mode === 'exact') result = text === matcher.value; + if (matcher.mode === 'contains') result = text.includes(matcher.value); + if (matcher.mode === 'regex') result = new RegExp(matcher.value, matcher.flags || '').test(text); + if (matcher.mode === 'predicate') result = matcher.predicate!(text); + if (matcher.negated) return !result; + return result; + } + + private valueMatcher(matcher: Matcher): TextMatcher { + if (typeof matcher === 'function') return { mode: 'predicate', value: '', negated: false, predicate: matcher }; + if (matcher instanceof RegExp) return { mode: 'regex', value: matcher.source, negated: false, flags: matcher.flags }; + return { mode: 'exact', value: matcher, negated: false }; + } + + private tokenText(token: Token): string { + const value = token as any; + if (this.isComment(token)) return (value.raw as string).trim().replace(/^$/, '').trim(); + if (token.type === 'html') return value.raw || ''; + if (token.type === 'table') return [...(value.header || []).map((cell: any) => cell.text), ...(value.rows || []).flatMap((row: any) => row.map((cell: any) => cell.text))].join(', '); + return value.text || ''; + } + + private isComment(token: Token): boolean { + if (token.type !== 'html') return false; + return (((token as any).raw as string) || '').trimStart().startsWith(' + +## Plan + + + +para with comment + +
a block
+ +| Method | Path | +|--------|------| +| GET | /users | +`; + +describe('comment selector', () => { + it('matches block comments and not other html', () => { + expect(mdq(doc).query('comment').count()).toBe(2); + }); + + it('matches on the inner body so anchored patterns work', () => { + expect(mdq(doc).query('comment(/^test/)').count()).toBe(1); + }); + + it('exposes the inner body as node text, without the markers', () => { + expect(mdq(doc).query('comment[0]').nodes()[0].text).toBe('suite'); + }); + + it('keeps newlines inside a multi-line comment', () => { + expect(mdq(doc).query('comment(/^test/)').nodes()[0].text).toContain('\n'); + }); + + it('does not reach comments inline in a paragraph', () => { + expect(mdq(doc).query('comment(~"inline")').count()).toBe(0); + }); + + it('matches an exact single-line comment body', () => { + expect(mdq(doc).query('comment("suite")').count()).toBe(1); + }); +}); + +describe('html selector', () => { + it('matches every html block including comments', () => { + expect(mdq(doc).query('html').count()).toBe(3); + }); + + it('matches on raw text', () => { + expect(mdq(doc).query('html(~" { + it('honors an explicit i flag', () => { + expect(mdq('## Summary\n').query('h2(/^summary/i)').count()).toBe(1); + }); + + it('is case sensitive without the i flag', () => { + expect(mdq('## Summary\n').query('h2(/^summary/)').count()).toBe(0); + }); +}); + +describe('table text matching', () => { + it('matches cell content, not only headers', () => { + expect(mdq(doc).query('table(~"/users")').count()).toBe(1); + }); + + it('still matches header content', () => { + expect(mdq(doc).query('table(~"Method")').count()).toBe(1); + }); +}); + +describe('selector errors', () => { + it('throws on an unknown selector rather than matching nothing', () => { + expect(() => mdq(doc).query('secton("A")')).toThrow(MdqSelectorError); + }); + + it('reports where the problem is', () => { + try { + mdq(doc).query('h2("A") secton("B")'); + expect.unreachable(); + } catch (error: any) { + expect(error.index).toBe(8); + } + }); + + it('accepts a leading dot for jq muscle memory', () => { + expect(mdq('## A\n').query('.h2').count()).toBe(1); + }); +}); diff --git a/tests/unit/mdq/structural.test.ts b/tests/unit/mdq/structural.test.ts new file mode 100644 index 00000000..6e4ff3b4 --- /dev/null +++ b/tests/unit/mdq/structural.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest'; +import { mdq } from '../../../src/utils/mdq/query.ts'; + +const table = `| Method | Path | +|--------|------| +| GET | /users | +`; + +describe('addRow', () => { + it('appends a row and re-aligns every column', () => { + expect(mdq(table).query('table').addRow({ Method: 'POST', Path: '/sessions' }).toString()).toBe(['| Method | Path |', '| ------ | --------- |', '| GET | /users |', '| POST | /sessions |', ''].join('\n')); + }); + + it('round-trips through rows()', () => { + const out = mdq(table).query('table').addRow({ Method: 'POST', Path: '/sessions' }); + expect(mdq(out).query('table').rows()).toEqual([ + { Method: 'GET', Path: '/users' }, + { Method: 'POST', Path: '/sessions' }, + ]); + }); + + it('leaves a column blank when the object omits it', () => { + const out = mdq(table).query('table').addRow({ Method: 'PUT' }); + expect(mdq(out).query('table').rows()[1]).toEqual({ Method: 'PUT', Path: '' }); + }); + + it('ignores keys that are not columns', () => { + const out = mdq(table).query('table').addRow({ Method: 'PUT', Nope: 'x' }); + expect(mdq(out).query('table').rows()[1].Method).toBe('PUT'); + expect(mdq(out).query('table').text()).not.toContain('Nope'); + }); + + it('preserves column alignment markers', () => { + const aligned = '| A | B |\n|:--|--:|\n| 1 | 2 |\n'; + const out = mdq(aligned).query('table').addRow({ A: '3', B: '4' }).toString(); + expect(out).toContain(':--'); + expect(out).toContain('--:'); + }); + + it('throws on a non-table node', () => { + expect(() => mdq('para\n').query('paragraph').addRow({ a: 'b' })).toThrow(); + }); +}); + +describe('addItem', () => { + it('copies a dash marker', () => { + expect(mdq('- a\n- b\n').query('list').addItem('c').toString()).toBe('- a\n- b\n- c\n'); + }); + + it('copies a star marker', () => { + expect(mdq('* a\n* b\n').query('list').addItem('c').toString()).toBe('* a\n* b\n* c\n'); + }); + + it('continues an ordered list', () => { + expect(mdq('1. a\n2. b\n').query('list').addItem('c').toString()).toBe('1. a\n2. b\n3. c\n'); + }); + + it('preserves indentation', () => { + expect(mdq(' - a\n - b\n').query('list').addItem('c').toString()).toBe(' - a\n - b\n - c\n'); + }); + + it('throws on a non-list node', () => { + expect(() => mdq('para\n').query('paragraph').addItem('x')).toThrow(); + }); +}); diff --git a/tests/unit/mdq/sugar.test.ts b/tests/unit/mdq/sugar.test.ts new file mode 100644 index 00000000..145ed005 --- /dev/null +++ b/tests/unit/mdq/sugar.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from 'vitest'; +import { mdq } from '../../../src/utils/mdq/query.ts'; + +const doc = `## API "v2" + +| Method | Path | +|--------|------| +| GET | /users | + +## Settings + +- Option A +- Option B + + +`; + +describe('matchers as values', () => { + it('matches a string exactly', () => { + expect(mdq(doc).query('h2', 'Settings').count()).toBe(1); + expect(mdq(doc).query('h2', 'Setting').count()).toBe(0); + }); + + it('matches a RegExp honoring its flags', () => { + expect( + mdq(doc) + .query('h2', /^settings$/i) + .count() + ).toBe(1); + expect( + mdq(doc) + .query('h2', /^settings$/) + .count() + ).toBe(0); + }); + + it('matches a predicate', () => { + expect( + mdq(doc) + .query('h2', (t) => t.startsWith('API')) + .count() + ).toBe(1); + }); + + it('needs no escaping for a value containing quotes', () => { + expect(mdq(doc).query('h2', 'API "v2"').count()).toBe(1); + }); +}); + +describe('sugar', () => { + it('is equivalent to the query form', () => { + expect(mdq(doc).heading('Settings').text()).toBe(mdq(doc).query('heading', 'Settings').text()); + }); + + it('takes a depth option', () => { + expect(mdq(doc).section('Settings', { depth: 2 }).text()).toBe(mdq(doc).query('section2("Settings")').text()); + }); + + it('reads comments', () => { + expect(mdq(doc).comment(/^test/).count()).toBe(1); + }); + + it('chains from a Selection', () => { + expect(mdq(doc).section('API "v2"').table().rows()[0].Path).toBe('/users'); + }); + + it('takes no matcher', () => { + expect(mdq(doc).table().count()).toBe(1); + }); + + it('is available on a document returned by a write', () => { + expect(mdq(doc).table().replace('gone\n').heading().count()).toBe(2); + }); +}); + +describe('at and slice', () => { + it('selects by index like the DSL', () => { + expect(mdq(doc).heading().at(0).text()).toBe(mdq(doc).query('heading[0]').text()); + }); + + it('supports a negative index', () => { + expect(mdq(doc).heading().at(-1).text()).toContain('Settings'); + }); + + it('returns nothing for an out-of-bounds index', () => { + expect(mdq(doc).heading().at(99).count()).toBe(0); + expect(mdq(doc).heading().at(-99).count()).toBe(0); + }); + + it('slices like the DSL', () => { + expect(mdq(doc).item().slice(1).count()).toBe(1); + }); +}); + +describe('exists', () => { + it('is true when something matched', () => { + expect(mdq(doc).heading('Settings').exists()).toBe(true); + }); + + it('is false when nothing matched', () => { + expect(mdq(doc).heading('Nope').exists()).toBe(false); + }); +}); + +describe('canonical read names', () => { + it('rows matches the deprecated toJson', () => { + expect(mdq(doc).table().rows()).toEqual(mdq(doc).table().toJson()); + }); + + it('preceding matches the deprecated before', () => { + expect(mdq(doc).heading('Settings').preceding().text()).toBe(mdq(doc).query('heading("Settings")').before().text()); + }); + + it('following matches the deprecated after', () => { + expect(mdq(doc).heading('API "v2"').following().text()).toBe(mdq(doc).query('heading(~"API")').after().text()); + }); + + it('entries matches the deprecated keyValue', () => { + const block = mdq('> Container: .x\n').query('blockquote[0]'); + expect(block.entries()).toEqual(block.keyValue()); + }); +}); + +describe('stringification', () => { + const src = '# A\n\ntext\n'; + + it('stringifies to the whole document, matched or not', () => { + expect(String(mdq(src))).toBe(src); + expect(String(mdq(src).query('paragraph'))).toBe(src); + }); + + it('gives the matched markdown through text()', () => { + expect(mdq(src).query('paragraph').text()).toBe('text\n'); + }); + + it('survives a round trip back through mdq()', () => { + expect(mdq(mdq(src).query('paragraph')).toString()).toBe(src); + }); +}); diff --git a/tests/unit/research-parser-pagination.test.ts b/tests/unit/research-parser-pagination.test.ts index 2c7e9f64..b07420ed 100644 --- a/tests/unit/research-parser-pagination.test.ts +++ b/tests/unit/research-parser-pagination.test.ts @@ -62,7 +62,7 @@ describe('pagination line', () => { }); describe('rewriting an entry', () => { - const rewrite = (markdown: string, key: string, value: string) => mdq(markdown).query('section2(~"Menu")').query('blockquote[0]').setKeyValue(key, value); + const rewrite = (markdown: string, key: string, value: string) => mdq(markdown).query('section2(~"Menu")').query('blockquote[0]').setKeyValue(key, value).toString(); it('leaves the blockquote readable', () => { const markdown = `## Menu\n\n> Container: '.old'\n\n| Element | ARIA | CSS | eidx |\n`;