From e5cf18bfabd679a32bfe45c04a04abdfc16942a8 Mon Sep 17 00:00:00 2001 From: edochi Date: Thu, 20 Aug 2026 19:26:57 +0200 Subject: [PATCH 1/3] chore: hard-wrap markdown at 80 cols via prettier + pre-commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add prettier (proseWrap: always, printWidth: 80) wired through a local pre-commit hook, and tune markdownlint's MD013 to match so the two agree. Tables, fenced code blocks, headings and long URLs are left over-length by design — MD013 disables those three checks and is non-strict for unbreakable lines. embeddedLanguageFormatting is off so prettier never rewrites the contents of a fence. Corpora and fixtures are excluded from both tools: prettier rewrites YAML frontmatter and does not recognise TOML (+++) or bare-brace JSON frontmatter, which would mangle example_kb, assets/demo_kb and the multi-format test fixtures. The generated CHANGELOG.md is excluded to avoid fighting cog. AGENTS.md is reformatted by the new hook; the rest of the tree is left for a separate bulk pass. --- .markdownlint-cli2.yaml | 24 +++++++++ .pre-commit-config.yaml | 22 ++++++++ .prettierignore | 13 +++++ .prettierrc.yaml | 14 +++++ AGENTS.md | 110 +++++++++++++++++++++++++++++++--------- 5 files changed, 158 insertions(+), 25 deletions(-) create mode 100644 .markdownlint-cli2.yaml create mode 100644 .pre-commit-config.yaml create mode 100644 .prettierignore create mode 100644 .prettierrc.yaml diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml new file mode 100644 index 0000000..cbaa292 --- /dev/null +++ b/.markdownlint-cli2.yaml @@ -0,0 +1,24 @@ +# markdownlint-cli2 config. Read by editors — LazyVim runs markdownlint-cli2 +# via nvim-lint over stdin, resolving config from cwd upward. Not wired into +# pre-commit: prettier writes the format, this only reports. The two must agree +# on width, so line_length here tracks printWidth in .prettierrc.yaml. + +config: + # 80-column limit, matching prettier's printWidth. The three exclusions are + # exactly what prettier's proseWrap deliberately will not wrap; without them + # markdownlint flags lines prettier just produced. + MD013: + line_length: 80 + code_blocks: false + tables: false + headings: false + +ignores: + # Mirrors .prettierignore — nothing prettier skips should be linted, or the + # editor lights up on files we deliberately do not format. + - "example_kb/**" + - "assets/demo_kb/**" + - "crates/mdvs/tests/fixtures/**" + - "crates/mdvs/CHANGELOG.md" + - "target/**" + - "node_modules/**" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..529f81a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +# Run `uv tool install pre-commit` (not `uvx` — `pre-commit install` bakes the +# interpreter path into .git/hooks/pre-commit, and a uvx cache entry can be +# pruned out from under it), then `pre-commit install` once per clone. +# +# Node and prettier are provisioned by pre-commit into ~/.cache/pre-commit; +# nothing needs to be installed globally. +repos: + - repo: local + hooks: + - id: prettier + name: prettier (markdown) + entry: prettier --write + language: node + additional_dependencies: ["prettier@3.6.2"] + types: [markdown] + exclude: | + (?x)^( + example_kb/| + assets/demo_kb/| + crates/mdvs/tests/fixtures/| + crates/mdvs/CHANGELOG\.md$ + ) diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..f5d77ec --- /dev/null +++ b/.prettierignore @@ -0,0 +1,13 @@ +# Corpora and fixtures: the exact frontmatter bytes are the thing under test. +# Prettier rewrites YAML frontmatter (quote style, indentation, flow-seq +# spacing) and does not recognise TOML (+++) or bare-brace JSON frontmatter at +# all, so it would mangle the multi-format fixtures. +example_kb/ +assets/demo_kb/ +crates/mdvs/tests/fixtures/ + +# Generated by cocogitto on every release bump — reformatting fights regen. +crates/mdvs/CHANGELOG.md + +# Build output. +target/ diff --git a/.prettierrc.yaml b/.prettierrc.yaml new file mode 100644 index 0000000..7a8b9e0 --- /dev/null +++ b/.prettierrc.yaml @@ -0,0 +1,14 @@ +# Prettier config — markdown only (no JS/TS/CSS in this repo). +# Paired with .markdownlint-cli2.yaml: prettier does the wrapping, markdownlint +# checks it. The two must agree on line_length / printWidth or they fight. + +# Hard-wrap prose at 80 columns. Only paragraphs, list-item text, and +# blockquotes are wrapped — tables, code fences, and headings are left alone. +proseWrap: always +printWidth: 80 + +# Freeze fenced code blocks. Prettier's default ("auto") reformats fences whose +# language it can parse — json, yaml, js, css — which would silently rewrite +# doc examples. rust/bash/toml fences have no parser and are untouched either +# way; this makes the guarantee uniform. +embeddedLanguageFormatting: "off" diff --git a/AGENTS.md b/AGENTS.md index cd67e3a..1bfbd0d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,20 +1,34 @@ # AGENTS.md -Guidance for AI coding agents working with this repository. Provider-agnostic — symlinked from `CLAUDE.md`, `.cursorrules`, and similar so any agent reads the same instructions. +Guidance for AI coding agents working with this repository. Provider-agnostic — +symlinked from `CLAUDE.md`, `.cursorrules`, and similar so any agent reads the +same instructions. ## Project Overview -mdvs (Markdown Validation & Search) is a Rust CLI that treats markdown directories as databases — schema inference, frontmatter validation, and semantic/full-text/hybrid search with SQL filtering. Single binary, no external services. Design specs live in `docs/spec/`, user-facing docs in `book/`. +mdvs (Markdown Validation & Search) is a Rust CLI that treats markdown +directories as databases — schema inference, frontmatter validation, and +semantic/full-text/hybrid search with SQL filtering. Single binary, no external +services. Design specs live in `docs/spec/`, user-facing docs in `book/`. ## Git Rules -**Never push directly to `main`.** All work goes through feature branches and PRs. One branch per TODO or feature (`feat/description`, `fix/description`, `docs/description`). Regular merge (not squash). Always ask the user before creating a branch. +**Never push directly to `main`.** All work goes through feature branches and +PRs. One branch per TODO or feature (`feat/description`, `fix/description`, +`docs/description`). Regular merge (not squash). Always ask the user before +creating a branch. -**Releases** go through a `release/v` branch + PR, then a tag push on main triggers the build. +**Releases** go through a `release/v` branch + PR, then a tag push on +main triggers the build. -**NEVER commit or push unless the user explicitly asks.** No autonomous commits. No "let me commit this" — wait for the user to say "commit" or "commit and push". This is non-negotiable. +**NEVER commit or push unless the user explicitly asks.** No autonomous commits. +No "let me commit this" — wait for the user to say "commit" or "commit and +push". This is non-negotiable. -**Use conventional commits.** A `commit-msg` hook (cocogitto) enforces the format `[optional scope]: ` locally. Types: `feat`, `fix`, `refactor`, `docs`, `test`, `chore`, `ci`, `perf`, `style`. See `docs/spec/cocogitto.md` for the full guide. +**Use conventional commits.** A `commit-msg` hook (cocogitto) enforces the +format `[optional scope]: ` locally. Types: `feat`, `fix`, +`refactor`, `docs`, `test`, `chore`, `ci`, `perf`, `style`. See +`docs/spec/cocogitto.md` for the full guide. ## Build & Verify @@ -28,32 +42,61 @@ cargo clippy --all-targets --features testing-mocks # lint (matches CI) cargo fmt # format ``` -**Always use `cargo clippy --all-targets --features testing-mocks`** — plain `cargo clippy` misses warnings in test code and the mock feature gate. **Run `cargo fmt` after `cargo clippy`.** +**Always use `cargo clippy --all-targets --features testing-mocks`** — plain +`cargo clippy` misses warnings in test code and the mock feature gate. **Run +`cargo fmt` after `cargo clippy`.** -The `testing-mocks` feature gates the deterministic `MockEmbedder` (`provider = "mock"` in `mdvs.toml`). It is off in production binaries (`cargo install`); `cargo test` and `cargo clippy` see it via `cfg(test)`. Real-model tests are marked `#[ignore]` so the fast lane stays hermetic — no Hugging Face network calls. See TODO-0184. +The `testing-mocks` feature gates the deterministic `MockEmbedder` +(`provider = "mock"` in `mdvs.toml`). It is off in production binaries +(`cargo install`); `cargo test` and `cargo clippy` see it via `cfg(test)`. +Real-model tests are marked `#[ignore]` so the fast lane stays hermetic — no +Hugging Face network calls. See TODO-0184. ## Markdown conventions -- **No hard-wrap.** Markdown files in this repo are not wrapped at a column limit — write each paragraph on a single line and let the editor soft-wrap. This applies to README, `book/`, `docs/spec/`, TODO files, PR descriptions stored as files, and any other `.md` in the tree. -- Don't reference scratch or gitignored folders in committed markdown files — keep references in committed docs limited to paths that survive a fresh clone. +- **Hard-wrap at 80 columns.** Markdown prose in this repo is wrapped at 80 + characters. Don't wrap by hand — `prettier` does it, configured in + `.prettierrc.yaml` and enforced by the `pre-commit` hook. Tables, fenced code + blocks, headings, and long URLs are deliberately left over-length; `MD013` in + `.markdownlint-cli2.yaml` is configured to match. This applies to README, + `book/`, `docs/spec/`, TODO files, and any other `.md` in the tree except the + paths in `.prettierignore` (`example_kb/`, `assets/demo_kb/`, test fixtures, + and the generated `CHANGELOG.md`), whose exact bytes are fixtures. +- **Setup:** `uv tool install pre-commit && pre-commit install`. Use + `uv tool install`, not `uvx` — `pre-commit install` bakes the interpreter path + into `.git/hooks/pre-commit` and a uvx cache entry can be pruned away. Node + and prettier are provisioned by pre-commit; no global npm install needed. +- Don't reference scratch or gitignored folders in committed markdown files — + keep references in committed docs limited to paths that survive a fresh clone. ## Architectural Invariants These survive across refactors; reach for them when in doubt: -- **Enum dispatch, no `dyn Trait`.** Backends, embedders, value stages, constraint kinds, search modes, outcomes are all enums with exhaustive matches. Adding a variant must update every match. -- **Two layers.** Validation (`init` / `update` / `check`) needs no embedding model. Search (`build` / `search`) needs the model + the Lance index in `.mdvs/`. Validation must stand alone. -- **Strict types.** `FieldType::String` rejects bools and numbers. Coercion is the preprocessor pipeline's job (`[[fields.field]].preprocess`), not the schema's. -- **Single source of truth.** `mdvs.toml` is the schema (committed); `.mdvs/` is build state (gitignored, recreatable). No lock file. -- **Build includes check.** Validation runs before embedding; violations abort the build. +- **Enum dispatch, no `dyn Trait`.** Backends, embedders, value stages, + constraint kinds, search modes, outcomes are all enums with exhaustive + matches. Adding a variant must update every match. +- **Two layers.** Validation (`init` / `update` / `check`) needs no embedding + model. Search (`build` / `search`) needs the model + the Lance index in + `.mdvs/`. Validation must stand alone. +- **Strict types.** `FieldType::String` rejects bools and numbers. Coercion is + the preprocessor pipeline's job (`[[fields.field]].preprocess`), not the + schema's. +- **Single source of truth.** `mdvs.toml` is the schema (committed); `.mdvs/` is + build state (gitignored, recreatable). No lock file. +- **Build includes check.** Validation runs before embedding; violations abort + the build. - **No interactive prompts.** Every flow is config-driven until 1.0. ## Pointers -- Architecture, data pipeline, storage layout, design decisions: `docs/spec/architecture.md`, `docs/spec/storage.md`, per-command pages under `docs/spec/commands/`. +- Architecture, data pipeline, storage layout, design decisions: + `docs/spec/architecture.md`, `docs/spec/storage.md`, per-command pages under + `docs/spec/commands/`. - Commands and flags: `mdvs --help` and `docs/spec/commands/`. - Dependencies and their roles: comments in `crates/mdvs/Cargo.toml`. -- Skills (agent workflows): `.claude/skills/` — invoke via `Skill` for `commit`, `todo`, `rust`, `spec`, `code-editing`, etc. +- Skills (agent workflows): `.claude/skills/` — invoke via `Skill` for `commit`, + `todo`, `rust`, `spec`, `code-editing`, etc. - TODOs (in-flight + done): `docs/spec/todos/index.md`. String ``` -Each arrow means "widens to." **String is the top type** — every type eventually reaches it. +Each arrow means "widens to." **String is the top type** — every type eventually +reaches it. -The one special case is Integer → Float: integers widen to floats (not directly to String) because the conversion is lossless. Date and DateTime have no internal cross-promotion — mixed `Date + DateTime` observations widen to String (the two shapes are disjoint). +The one special case is Integer → Float: integers widen to floats (not directly +to String) because the conversion is lossless. Date and DateTime have no +internal cross-promotion — mixed `Date + DateTime` observations widen to String +(the two shapes are disjoint). Two same-category combinations widen internally instead of jumping to String: -- **Array + Array** — element types are widened recursively (e.g., `Array(Integer)` + `Array(String)` → `Array(String)`) -- **Object + Object** — at the leaf level: each dotted path's type is widened independently across files. A file with `cal.wave = 850` (Integer) and another with `cal.wave = 632.8` (Float) yields `cal.wave: Float`. New leaf paths in some files are added to the schema; leaves absent from some files affect nullability/required-globs naturally. -Everything else (Boolean + any other type, Array + scalar, Object + scalar) widens to String. The one exception is **Array containing Object** — `Array(Object{...})` isn't representable on disk, so inference drops the field with a warning instead of widening to String (see [Arrays of structured items](#arrays-of-structured-items)). +- **Array + Array** — element types are widened recursively (e.g., + `Array(Integer)` + `Array(String)` → `Array(String)`) +- **Object + Object** — at the leaf level: each dotted path's type is widened + independently across files. A file with `cal.wave = 850` (Integer) and another + with `cal.wave = 632.8` (Float) yields `cal.wave: Float`. New leaf paths in + some files are added to the schema; leaves absent from some files affect + nullability/required-globs naturally. + +Everything else (Boolean + any other type, Array + scalar, Object + scalar) +widens to String. The one exception is **Array containing Object** — +`Array(Object{...})` isn't representable on disk, so inference drops the field +with a warning instead of widening to String (see +[Arrays of structured items](#arrays-of-structured-items)). ## Type widening in practice -When mdvs scans your files and the same field has different types, it picks the **least upper bound** — the most specific type that covers all observed values. +When mdvs scans your files and the same field has different types, it picks the +**least upper bound** — the most specific type that covers all observed values. ### Integer + Float → Float @@ -227,7 +289,8 @@ wavelength_nm: 632.8 # Float wavelength_nm: 780.0 # Float ``` -Result: `wavelength_nm` is inferred as **Float**. The integer `850` is safely represented as a float. +Result: `wavelength_nm` is inferred as **Float**. The integer `850` is safely +represented as a float. ### Integer + String → String @@ -241,17 +304,23 @@ priority: 1 # Integer priority: high # String ``` -Result: `priority` is inferred as **String**. There's no numeric type that can hold `"high"`, so mdvs widens to String. +Result: `priority` is inferred as **String**. There's no numeric type that can +hold `"high"`, so mdvs widens to String. ### Boolean + any non-Boolean → String -If the same field is `true` in one file and `3` in another, there's no numeric or boolean type that can hold both. The result is String. +If the same field is `true` in one file and `3` in another, there's no numeric +or boolean type that can hold both. The result is String. -This doesn't happen in `example_kb` because booleans (`draft`) are used consistently — but it's a common mistake in organically grown vaults where someone writes `draft: yes` (String) instead of `draft: true` (Boolean). +This doesn't happen in `example_kb` because booleans (`draft`) are used +consistently — but it's a common mistake in organically grown vaults where +someone writes `draft: yes` (String) instead of `draft: true` (Boolean). ### Date and DateTime inference -A string is inferred as `Date` or `DateTime` when **every observation** across all files matches the RFC 3339 shape AND parses as a real value. A single non-matching value downgrades the whole field to String. +A string is inferred as `Date` or `DateTime` when **every observation** across +all files matches the RFC 3339 shape AND parses as a real value. A single +non-matching value downgrades the whole field to String. Pure-date observations across files: @@ -275,7 +344,8 @@ joined: 2023-02-01 joined: "see HR records" # not a date ``` -Result: `joined` widens to **String** — the second observation can't be typed as Date, and `Date + String → String` is the widening rule. +Result: `joined` widens to **String** — the second observation can't be typed as +Date, and `Date + String → String` is the widening rule. Same logic for invalid calendar dates: @@ -287,7 +357,9 @@ published: 2024-06-01 published: 2024-13-01 # invalid month — typed String per-value ``` -Result: `published` widens to **String**. The typo gets silently absorbed into String typing; the user only catches it via a `WrongType` violation if they manually set `type = "Date"` in `mdvs.toml`. +Result: `published` widens to **String**. The typo gets silently absorbed into +String typing; the user only catches it via a `WrongType` violation if they +manually set `type = "Date"` in `mdvs.toml`. Date + DateTime are cross-shape — never auto-promote: @@ -299,11 +371,13 @@ when: 2024-01-15 # Date when: 2024-01-15T14:30:00Z # DateTime ``` -Result: `when` widens to **String**. Pick one shape consistently to get a typed field. +Result: `when` widens to **String**. Pick one shape consistently to get a typed +field. ### Array element widening -The `tags` field is a string array in most files, but one file accidentally used integers: +The `tags` field is a string array in most files, but one file accidentally used +integers: ```yaml # projects/alpha/overview.md @@ -318,13 +392,18 @@ tags: - 3 # Array(Integer) ``` -Result: `tags` is inferred as **`Array(String)`**. The array element types (String vs Integer) are widened to String, giving `Array(String)`. +Result: `tags` is inferred as **`Array(String)`**. The array element types +(String vs Integer) are widened to String, giving `Array(String)`. ### Object leaf merging (dotted-name flattening) -When two files have nested keys at the same paths, each leaf is inferred independently. New leaves seen in one file but not another are added to the schema; their `required` glob naturally narrows to just the files that contain them. +When two files have nested keys at the same paths, each leaf is inferred +independently. New leaves seen in one file but not another are added to the +schema; their `required` glob naturally narrows to just the files that contain +them. -In `example_kb`, the `calibration` object appears in two experiment files with different structures: +In `example_kb`, the `calibration` object appears in two experiment files with +different structures: ```yaml # experiment-1.md (simpler calibration, integer values) @@ -371,45 +450,58 @@ preprocess = ["widen-int-to-float"] ``` What happened: -- `calibration.baseline.wavelength` seen as both Integer (850) and Float (632.8) → widened to Float with `widen-int-to-float` preprocessor recording the mix -- `calibration.baseline.intensity` similar: Integer (1) + Float (0.95) → Float with the preprocessor -- `calibration.baseline.notes` only in experiment-1 → still inferred as String (with a `required` glob narrowed to just the files that have it) + +- `calibration.baseline.wavelength` seen as both Integer (850) and Float (632.8) + → widened to Float with `widen-int-to-float` preprocessor recording the mix +- `calibration.baseline.intensity` similar: Integer (1) + Float (0.95) → Float + with the preprocessor +- `calibration.baseline.notes` only in experiment-1 → still inferred as String + (with a `required` glob narrowed to just the files that have it) - `calibration.adjusted.*` only in experiment-2 → inferred from that file alone -The user-facing schema is flat, but its semantics still match the YAML's nested shape. Validation, storage, and `--where` queries all operate on the natural nested structure — the dotted-name form is purely a `mdvs.toml` UX choice. +The user-facing schema is flat, but its semantics still match the YAML's nested +shape. Validation, storage, and `--where` queries all operate on the natural +nested structure — the dotted-name form is purely a `mdvs.toml` UX choice. ## The full widening matrix Every possible combination of types and its result: -| | Boolean | Integer | Float | String | Date | DateTime | Array | Object | -|---|---|---|---|---|---|---|---|---| -| **Boolean** | Boolean | String | String | String | String | String | String | String | -| **Integer** | String | Integer | **Float** | String | String | String | String | String | -| **Float** | String | **Float** | Float | String | String | String | String | String | -| **String** | String | String | String | String | String | String | String | String | -| **Date** | String | String | String | String | Date | String | String | String | -| **DateTime** | String | String | String | String | String | DateTime | String | String | -| **Array** | String | String | String | String | String | String | Array\* | dropped\*\* | -| **Object** | String | String | String | String | String | String | dropped\*\* | Object\* | +| | Boolean | Integer | Float | String | Date | DateTime | Array | Object | +| ------------ | ------- | --------- | --------- | ------ | ------ | -------- | ----------- | ----------- | +| **Boolean** | Boolean | String | String | String | String | String | String | String | +| **Integer** | String | Integer | **Float** | String | String | String | String | String | +| **Float** | String | **Float** | Float | String | String | String | String | String | +| **String** | String | String | String | String | String | String | String | String | +| **Date** | String | String | String | String | Date | String | String | String | +| **DateTime** | String | String | String | String | String | DateTime | String | String | +| **Array** | String | String | String | String | String | String | Array\* | dropped\*\* | +| **Object** | String | String | String | String | String | String | dropped\*\* | Object\* | \* Array + Array: element types are widened recursively. -\* Object + Object: not a top-level on-disk type. Nested Objects in YAML flatten to dotted-name leaves before widening; each leaf path is widened independently. +\* Object + Object: not a top-level on-disk type. Nested Objects in YAML flatten +to dotted-name leaves before widening; each leaf path is widened independently. -\*\* Inference observed Array(Object{...}) — not representable on disk in v0. The field is dropped from the schema and a warning is emitted (see [Arrays of structured items](#arrays-of-structured-items)). +\*\* Inference observed Array(Object{...}) — not representable on disk in v0. +The field is dropped from the schema and a warning is emitted (see +[Arrays of structured items](#arrays-of-structured-items)). -Date and DateTime are cross-shape — they never auto-promote into each other. The single non-trivial pair is `Date + DateTime → String`. +Date and DateTime are cross-shape — they never auto-promote into each other. The +single non-trivial pair is `Date + DateTime → String`. The matrix is symmetric — `widen(A, B)` always equals `widen(B, A)`. ## Nullable -Separately from the type, mdvs tracks whether `null` was observed for a field. This is shown as a `?` suffix in output — e.g., `Float?` means "Float, but sometimes null." +Separately from the type, mdvs tracks whether `null` was observed for a field. +This is shown as a `?` suffix in output — e.g., `Float?` means "Float, but +sometimes null." ### How it works -In `example_kb`, the `drift_rate` field is Float in two experiment files but null in a third: +In `example_kb`, the `drift_rate` field is Float in two experiment files but +null in a third: ```yaml # experiment-1.md @@ -422,7 +514,8 @@ drift_rate: null # sensor malfunction — Giulia discarded the data drift_rate: 0.012 # Float ``` -Result: `drift_rate` is inferred as **Float?** — the type is Float (null doesn't affect the type), and `nullable` is set to true. +Result: `drift_rate` is inferred as **Float?** — the type is Float (null doesn't +affect the type), and `nullable` is set to true. ### Null-only fields @@ -440,13 +533,18 @@ Result: `review_score` is inferred as **String?**. - Null is **transparent** in widening — it doesn't affect the inferred type - Null-only fields default to String (the safest fallback) - `nullable` is a separate boolean, not part of the type itself -- In validation: null values skip type checks, but a non-nullable required field with a null value triggers a `NullNotAllowed` violation (see [Validation](./validation.md)) +- In validation: null values skip type checks, but a non-nullable required field + with a null value triggers a `NullNotAllowed` violation (see + [Validation](./validation.md)) ## Widening and preprocessors -Widening picks the type. **Preprocessors** are how the schema declares what coercions were needed to get there. Inference auto-populates them — you rarely write them by hand. +Widening picks the type. **Preprocessors** are how the schema declares what +coercions were needed to get there. Inference auto-populates them — you rarely +write them by hand. -When inference observes a field as a mix of types (some files have `priority: 1`, others `priority: high`), it widens to `String` and writes: +When inference observes a field as a mix of types (some files have +`priority: 1`, others `priority: high`), it widens to `String` and writes: ```toml [[fields.field]] @@ -455,27 +553,45 @@ type = "String" preprocess = ["coerce-to-string"] ``` -The `coerce-to-string` entry tells validation: "before checking this value is a string, serialize whatever you find to its JSON representation." Without it, the field is strict — integers and booleans fail validation. +The `coerce-to-string` entry tells validation: "before checking this value is a +string, serialize whatever you find to its JSON representation." Without it, the +field is strict — integers and booleans fail validation. -Same for Float: a mix of `5` and `5.0` widens to `Float` with `preprocess = ["widen-int-to-float"]`. Without it, integers fail the float check. +Same for Float: a mix of `5` and `5.0` widens to `Float` with +`preprocess = ["widen-int-to-float"]`. Without it, integers fail the float +check. The two built-in Stage 2 preprocessors: -| Preprocessor | Applies to | Effect | -|---|---|---| -| `coerce-to-string` | `String`, `Array(String)` | Serialize non-strings to their JSON string representation before validation | -| `widen-int-to-float` | `Float`, `Array(Float)` | Treat integer values as their float equivalent | +| Preprocessor | Applies to | Effect | +| -------------------- | ------------------------- | --------------------------------------------------------------------------- | +| `coerce-to-string` | `String`, `Array(String)` | Serialize non-strings to their JSON string representation before validation | +| `widen-int-to-float` | `Float`, `Array(Float)` | Treat integer values as their float equivalent | -**`preprocess = []` means strict.** If you delete a preprocessor from `mdvs.toml`, the field rejects values that would have been coerced. Conversely, you can hand-add a preprocessor to a strict-inferred field if you want to accept type variation. +**`preprocess = []` means strict.** If you delete a preprocessor from +`mdvs.toml`, the field rejects values that would have been coerced. Conversely, +you can hand-add a preprocessor to a strict-inferred field if you want to accept +type variation. -**No preprocessor applies to `Date` or `DateTime`.** Those types are strict by design — values either parse as RFC 3339 or they don't. There is no `parse-loose-date` opt-in; non-ISO formats fall back to String (and the user can add a `pattern` constraint if they want a custom shape). +**No preprocessor applies to `Date` or `DateTime`.** Those types are strict by +design — values either parse as RFC 3339 or they don't. There is no +`parse-loose-date` opt-in; non-ISO formats fall back to String (and the user can +add a `pattern` constraint if they want a custom shape). -**In storage** — when validation accepts a coerced value, the coerced form is what gets stored. A `priority: 1` value with `coerce-to-string` becomes `"1"` in the search index. No data is silently dropped. +**In storage** — when validation accepts a coerced value, the coerced form is +what gets stored. A `priority: 1` value with `coerce-to-string` becomes `"1"` in +the search index. No data is silently dropped. -Re-run `mdvs update reinfer ` to refresh both the inferred type and the inferred preprocessors after editing source files. +Re-run `mdvs update reinfer ` to refresh both the inferred type and the +inferred preprocessors after editing source files. ## Edge cases -- **Empty arrays** `[]` default to **`Array(String)`** — if real values are added later, the field must be re-inferred with `mdvs update reinfer ` to pick up the new element type -- **Empty frontmatter** (`---` followed immediately by `---`) is a file with zero fields — not a bare file. It still counts as "having frontmatter" for inference purposes. -- **Bare files** (no `---` fences at all) are handled differently — see [Schema Inference](./schema.md) +- **Empty arrays** `[]` default to **`Array(String)`** — if real values are + added later, the field must be re-inferred with `mdvs update reinfer ` + to pick up the new element type +- **Empty frontmatter** (`---` followed immediately by `---`) is a file with + zero fields — not a bare file. It still counts as "having frontmatter" for + inference purposes. +- **Bare files** (no `---` fences at all) are handled differently — see + [Schema Inference](./schema.md) diff --git a/book/src/concepts/validation.md b/book/src/concepts/validation.md index 333348d..669cbeb 100644 --- a/book/src/concepts/validation.md +++ b/book/src/concepts/validation.md @@ -1,126 +1,200 @@ # Validation -`mdvs check` validates every file's frontmatter against the schema in `mdvs.toml`. It's read-only and produces no side effects — it just tells you what's wrong. The output is byte-stable across runs: violations are sorted by `(field, kind, rule)` and the files within each violation are sorted by path, so CI tools that diff `mdvs check` output across runs get a clean comparison regardless of file-walking order. +`mdvs check` validates every file's frontmatter against the schema in +`mdvs.toml`. It's read-only and produces no side effects — it just tells you +what's wrong. The output is byte-stable across runs: violations are sorted by +`(field, kind, rule)` and the files within each violation are sorted by path, so +CI tools that diff `mdvs check` output across runs get a clean comparison +regardless of file-walking order. ## The seven violations -| Violation | Meaning | -|---|---| -| `WrongType` | The value doesn't match the declared `type` (or fails a `pattern` regex) | -| `Disallowed` | The field appears in a file outside its `allowed` paths | -| `MissingRequired` | A file matches a `required` glob but doesn't have the field | -| `NullNotAllowed` | The field is present but `null`, and `nullable` is `false` | -| `InvalidCategory` | The value is not in the field's declared `categories` | -| `OutOfRange` | A numeric value violates `min`/`max`, or a length violates `min_length`/`max_length` | +| Violation | Meaning | +| ---------------------------- | ---------------------------------------------------------------------------------------------------- | +| `WrongType` | The value doesn't match the declared `type` (or fails a `pattern` regex) | +| `Disallowed` | The field appears in a file outside its `allowed` paths | +| `MissingRequired` | A file matches a `required` glob but doesn't have the field | +| `NullNotAllowed` | The field is present but `null`, and `nullable` is `false` | +| `InvalidCategory` | The value is not in the field's declared `categories` | +| `OutOfRange` | A numeric value violates `min`/`max`, or a length violates `min_length`/`max_length` | | `FrontmatterUnrepresentable` | The file's frontmatter can't be represented as JSON (NaN/inf, non-string keys, non-object top-level) | ### WrongType -Fires when a value doesn't match the declared type. If `convergence_ms` is declared as `Boolean` but a file has `convergence_ms: 42`, the integer value fails the boolean check. +Fires when a value doesn't match the declared type. If `convergence_ms` is +declared as `Boolean` but a file has `convergence_ms: 42`, the integer value +fails the boolean check. -This violation has two important leniencies — see [Type checking rules](#type-checking-rules) below. +This violation has two important leniencies — see +[Type checking rules](#type-checking-rules) below. ### Disallowed -Fires when a field appears in a file whose path doesn't match any of the field's `allowed` globs. For example, if `firmware_version` has `allowed = ["people/interns/**"]` but appears in `people/remo.md`, that file is outside the allowed paths. +Fires when a field appears in a file whose path doesn't match any of the field's +`allowed` globs. For example, if `firmware_version` has +`allowed = ["people/interns/**"]` but appears in `people/remo.md`, that file is +outside the allowed paths. ### MissingRequired -Fires when a file's path matches one of the field's `required` globs, but the file doesn't contain that field at all. +Fires when a file's path matches one of the field's `required` globs, but the +file doesn't contain that field at all. -For example, if `observation_notes` has `required = ["projects/alpha/notes/**"]`, then every file under `projects/alpha/notes/` must have it. Files that don't → `MissingRequired`. +For example, if `observation_notes` has +`required = ["projects/alpha/notes/**"]`, then every file under +`projects/alpha/notes/` must have it. Files that don't → `MissingRequired`. ### NullNotAllowed -Fires when a field is present with an explicit `null` value, but `nullable` is `false`. For example, if `drift_rate` has `nullable = false` and a file has `drift_rate: null`. +Fires when a field is present with an explicit `null` value, but `nullable` is +`false`. For example, if `drift_rate` has `nullable = false` and a file has +`drift_rate: null`. -This is distinct from a missing field — see [Null vs absent](#null-vs-absent) below. +This is distinct from a missing field — see [Null vs absent](#null-vs-absent) +below. ### InvalidCategory -Fires when a field has a `categories` constraint and the value is not in the declared list. For example, if `status` has `categories = ["draft", "published", "archived"]` and a file has `status: pending`, the value `"pending"` is not in the list. +Fires when a field has a `categories` constraint and the value is not in the +declared list. For example, if `status` has +`categories = ["draft", "published", "archived"]` and a file has +`status: pending`, the value `"pending"` is not in the list. -For array fields, each element is checked individually. The violation detail lists the specific offending elements. +For array fields, each element is checked individually. The violation detail +lists the specific offending elements. -This check only runs on non-null values that pass the type check. If the value has the wrong type, only `WrongType` fires — `InvalidCategory` is skipped. If the value is null and the field is nullable, the category check is skipped entirely. +This check only runs on non-null values that pass the type check. If the value +has the wrong type, only `WrongType` fires — `InvalidCategory` is skipped. If +the value is null and the field is nullable, the category check is skipped +entirely. -See [Constraints](./constraints.md) for how categories are configured and auto-inferred. +See [Constraints](./constraints.md) for how categories are configured and +auto-inferred. ### OutOfRange Fires when a value violates a numeric or length bound: -- `min` / `max` on numeric fields — `rating: 7` with `min = 1, max = 5` is above `max`. -- `min_length` / `max_length` on string fields — `slug: "a"` with `min_length = 3` is too short. -- `min_items` / `max_items` on array fields (when emitted by inference) — applies to the array's length. +- `min` / `max` on numeric fields — `rating: 7` with `min = 1, max = 5` is above + `max`. +- `min_length` / `max_length` on string fields — `slug: "a"` with + `min_length = 3` is too short. +- `min_items` / `max_items` on array fields (when emitted by inference) — + applies to the array's length. -For array fields, numeric-element bounds are checked individually. The violation detail lists the specific offending elements or, for length checks, the actual length. +For array fields, numeric-element bounds are checked individually. The violation +detail lists the specific offending elements or, for length checks, the actual +length. -This check only runs on non-null values that pass the type check, same as `InvalidCategory`. +This check only runs on non-null values that pass the type check, same as +`InvalidCategory`. See [Constraints](./constraints.md) for how bounds are configured. ### FrontmatterUnrepresentable -Fires when a file's YAML frontmatter parses successfully but can't be represented as JSON. Causes include `NaN` / `inf` floats, non-string mapping keys, or a top-level value that isn't a mapping. The violation is reported at the document level with the sentinel field name ``. +Fires when a file's YAML frontmatter parses successfully but can't be +represented as JSON. Causes include `NaN` / `inf` floats, non-string mapping +keys, or a top-level value that isn't a mapping. The violation is reported at +the document level with the sentinel field name ``. -Pre-Wave-B mdvs silently dropped these files; they're now surfaced explicitly so the schema can't lie about what's actually in your vault. +Pre-Wave-B mdvs silently dropped these files; they're now surfaced explicitly so +the schema can't lie about what's actually in your vault. ## Type checking rules -Type checking is strict — a `String` field rejects integers, a `Boolean` field rejects strings, and so on. Two opt-in adjustments cover the common YAML pain points: - -**Preprocessors normalize before validation.** A field's `preprocess` array runs before jsonschema sees the value. Two built-ins: - -- `coerce-to-string` — non-string values (booleans, integers, arrays) are serialized to their JSON string representation, then validated as strings. Auto-inferred when the inferred type widened to `String` because of mixed-type observations. -- `widen-int-to-float` — integers are widened to equivalent floats. Auto-inferred when the inferred type widened to `Float` because some files used `5` and others `5.0`. Without it, a Float field rejects integer values. - -Fields with empty `preprocess` arrays are validated strictly — there are no implicit leniencies. See [Types & Widening](./types.md) for how inference picks the preprocessors. - -**Recursion.** Arrays check element types recursively — an `Array(Integer)` field rejects `["a", "b"]` because the string elements fail the Integer check. Nested frontmatter structure is validated per leaf: a config entry named `calibration.baseline.wavelength` is checked against the value at the corresponding nested path in the YAML. Missing intermediate Objects mean the leaf is absent — handled by the `MissingRequired` check. - -**Pattern.** A `pattern` constraint on a String field is enforced as a regex; pattern failures surface as `WrongType` (with detail naming the offending value). - -**Date and DateTime format validation.** `Date` and `DateTime` fields use JSON Schema's `format: date` / `format: date-time` keywords. Non-conforming values (invalid calendar dates, missing timezones, wrong separators) fire `WrongType` with a rule like `format date` or `format date-time`. See [Date and DateTime](./types.md#date-and-datetime) for the exact accepted shapes. +Type checking is strict — a `String` field rejects integers, a `Boolean` field +rejects strings, and so on. Two opt-in adjustments cover the common YAML pain +points: + +**Preprocessors normalize before validation.** A field's `preprocess` array runs +before jsonschema sees the value. Two built-ins: + +- `coerce-to-string` — non-string values (booleans, integers, arrays) are + serialized to their JSON string representation, then validated as strings. + Auto-inferred when the inferred type widened to `String` because of mixed-type + observations. +- `widen-int-to-float` — integers are widened to equivalent floats. + Auto-inferred when the inferred type widened to `Float` because some files + used `5` and others `5.0`. Without it, a Float field rejects integer values. + +Fields with empty `preprocess` arrays are validated strictly — there are no +implicit leniencies. See [Types & Widening](./types.md) for how inference picks +the preprocessors. + +**Recursion.** Arrays check element types recursively — an `Array(Integer)` +field rejects `["a", "b"]` because the string elements fail the Integer check. +Nested frontmatter structure is validated per leaf: a config entry named +`calibration.baseline.wavelength` is checked against the value at the +corresponding nested path in the YAML. Missing intermediate Objects mean the +leaf is absent — handled by the `MissingRequired` check. + +**Pattern.** A `pattern` constraint on a String field is enforced as a regex; +pattern failures surface as `WrongType` (with detail naming the offending +value). + +**Date and DateTime format validation.** `Date` and `DateTime` fields use JSON +Schema's `format: date` / `format: date-time` keywords. Non-conforming values +(invalid calendar dates, missing timezones, wrong separators) fire `WrongType` +with a rule like `format date` or `format date-time`. See +[Date and DateTime](./types.md#date-and-datetime) for the exact accepted shapes. ## Engine -Per-value validation runs through the `jsonschema` crate. mdvs translates `mdvs.toml`'s `[fields]` block into a JSON Schema 2020-12 document, compiles one validator per field, runs Stage 2 preprocessors, then validates each value. Errors from `jsonschema` are mapped exhaustively into the seven `ViolationKind`s above. +Per-value validation runs through the `jsonschema` crate. mdvs translates +`mdvs.toml`'s `[fields]` block into a JSON Schema 2020-12 document, compiles one +validator per field, runs Stage 2 preprocessors, then validates each value. +Errors from `jsonschema` are mapped exhaustively into the seven `ViolationKind`s +above. -One subtype check runs in Rust ahead of jsonschema: a `Float` field without `widen-int-to-float` rejects integer-backed values (`5` is rejected, `5.0` is accepted). JSON Schema's `"number"` accepts both — but YAML and TOML preserve the int/float distinction at parse time, and so does mdvs. +One subtype check runs in Rust ahead of jsonschema: a `Float` field without +`widen-int-to-float` rejects integer-backed values (`5` is rejected, `5.0` is +accepted). JSON Schema's `"number"` accepts both — but YAML and TOML preserve +the int/float distinction at parse time, and so does mdvs. ## Null handling Null interacts with validation in specific ways: -**The checks are independent.** A null value is checked like any other value — each violation type is evaluated separately: +**The checks are independent.** A null value is checked like any other value — +each violation type is evaluated separately: - **`WrongType`** — null is accepted by any type, so this never fires on null. -- **`Disallowed`** — the field is present (the key exists), so `Disallowed` fires if the path isn't in `allowed`. +- **`Disallowed`** — the field is present (the key exists), so `Disallowed` + fires if the path isn't in `allowed`. - **`MissingRequired`** — null counts as "present", so this never fires on null. - **`NullNotAllowed`** — fires when the value is null and `nullable = false`. -- **`InvalidCategory`** — null skips the category check (same as `WrongType`), so this never fires on null. -- **`OutOfRange`** — null skips the range check (same as `InvalidCategory`), so this never fires on null. +- **`InvalidCategory`** — null skips the category check (same as `WrongType`), + so this never fires on null. +- **`OutOfRange`** — null skips the range check (same as `InvalidCategory`), so + this never fires on null. -A single null field can trigger both `Disallowed` and `NullNotAllowed` at the same time. +A single null field can trigger both `Disallowed` and `NullNotAllowed` at the +same time. **Null vs absent.** These are different situations with different outcomes: -| Situation | Example | Result | -|---|---|---| -| Field is **absent** | File has no `drift_rate` key at all | `MissingRequired` (if path matches `required`) | -| Field is **null**, `nullable = true` | `drift_rate: null` | Passes | -| Field is **null**, `nullable = false` | `drift_rate: null` | `NullNotAllowed` | +| Situation | Example | Result | +| ------------------------------------- | ----------------------------------- | ---------------------------------------------- | +| Field is **absent** | File has no `drift_rate` key at all | `MissingRequired` (if path matches `required`) | +| Field is **null**, `nullable = true` | `drift_rate: null` | Passes | +| Field is **null**, `nullable = false` | `drift_rate: null` | `NullNotAllowed` | -A null value counts as "present" — the field key exists in the frontmatter, it just has no value. So null never triggers `MissingRequired`. An absent field is genuinely missing — it can trigger `MissingRequired` but never `NullNotAllowed`. +A null value counts as "present" — the field key exists in the frontmatter, it +just has no value. So null never triggers `MissingRequired`. An absent field is +genuinely missing — it can trigger `MissingRequired` but never `NullNotAllowed`. -> **Note:** In YAML, unquoted `null` is a null value, not the string `"null"`. To store the literal string, write `drift_rate: "null"` (with quotes). +> **Note:** In YAML, unquoted `null` is a null value, not the string `"null"`. +> To store the literal string, write `drift_rate: "null"` (with quotes). ## New fields -When `mdvs check` encounters a frontmatter field that isn't in `mdvs.toml` — neither constrained under `[[fields.field]]` nor listed in `ignore` — it reports it as a **new field**. +When `mdvs check` encounters a frontmatter field that isn't in `mdvs.toml` — +neither constrained under `[[fields.field]]` nor listed in `ignore` — it reports +it as a **new field**. -New fields are informational only. They don't count as violations and don't affect the exit code: +New fields are informational only. They don't count as violations and don't +affect the exit code: ``` Checked 43 files — no violations, 1 new field(s) @@ -130,24 +204,33 @@ Checked 43 files — no violations, 1 new field(s) ╰──────────────────────────────┴─────────────────────┴─────────────────────────╯ ``` -They're shown in the output so you know to either run `mdvs update` to add them to the schema, or add them to the `ignore` list. +They're shown in the output so you know to either run `mdvs update` to add them +to the schema, or add them to the `ignore` list. ## Bare files -When `include_bare_files = true` in `[scan]`, bare files (no frontmatter at all) are included in validation. Since they have no fields, they trigger `MissingRequired` for any `required` glob matching their path. +When `include_bare_files = true` in `[scan]`, bare files (no frontmatter at all) +are included in validation. Since they have no fields, they trigger +`MissingRequired` for any `required` glob matching their path. -For example, if `title` has `required = ["**"]` and `scratch.md` is a bare file, it triggers `MissingRequired` for `title`. This is often why the inferred schema uses narrower required globs — bare files at the root prevent `required = ["**"]` from being inferred for fields that don't appear in them. +For example, if `title` has `required = ["**"]` and `scratch.md` is a bare file, +it triggers `MissingRequired` for `title`. This is often why the inferred schema +uses narrower required globs — bare files at the root prevent +`required = ["**"]` from being inferred for fields that don't appear in them. ## Check and build -`mdvs build` runs the same validation internally before embedding. If any violations are found, build aborts — no dirty data reaches the index. The violations are the same ones `check` would report. +`mdvs build` runs the same validation internally before embedding. If any +violations are found, build aborts — no dirty data reaches the index. The +violations are the same ones `check` would report. -This means you can use `check` as a dry run before building, but you don't have to — build will catch the same problems. +This means you can use `check` as a dry run before building, but you don't have +to — build will catch the same problems. ## Exit codes -| Exit code | Meaning | -|---|---| -| 0 | No violations (new fields don't count) | -| 1 | One or more violations found | -| 2 | Scan or config error (couldn't run validation) | +| Exit code | Meaning | +| --------- | ---------------------------------------------- | +| 0 | No violations (new fields don't count) | +| 1 | One or more violations found | +| 2 | Scan or config error (couldn't run validation) | diff --git a/book/src/configuration.md b/book/src/configuration.md index 51849cc..dc78ddb 100644 --- a/book/src/configuration.md +++ b/book/src/configuration.md @@ -1,17 +1,21 @@ # Configuration -All configuration lives in `mdvs.toml`, created by [init](./commands/init.md) and updated by [update](./commands/update.md). This page is a complete reference of every section and field. +All configuration lives in `mdvs.toml`, created by [init](./commands/init.md) +and updated by [update](./commands/update.md). This page is a complete reference +of every section and field. ## Sections overview `mdvs.toml` has two groups of sections: **Validation** (always present): + - [`[scan]`](#scan) — file discovery - [`[check]`](#check) — check command settings - [`[fields]`](#fields) — field definitions and ignore list **Build & search** (written by `init`, model/chunking filled by first `build`): + - [`[embedding_model]`](#embedding_model) — model identity - [`[chunking]`](#chunking) — chunk sizing - [`[build]`](#build) — build workflow settings @@ -21,27 +25,37 @@ All configuration lives in `mdvs.toml`, created by [init](./commands/init.md) an These flags apply to all commands: -| Flag | Values | Default | Description | -|---|---|---|---| -| `-o`, `--output` | `pretty`, `markdown`, `json` | `pretty` | Output format. See [Output format selection](#output-format-selection) for the resolution chain when `-o` is omitted. | -| `-v`, `--verbose` | | | Show detailed output (pipeline steps, expanded records) | -| `--logs` | `info`, `debug`, `trace` | (none) | Enable diagnostic logging to stderr | +| Flag | Values | Default | Description | +| ----------------- | ---------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------- | +| `-o`, `--output` | `pretty`, `markdown`, `json` | `pretty` | Output format. See [Output format selection](#output-format-selection) for the resolution chain when `-o` is omitted. | +| `-v`, `--verbose` | | | Show detailed output (pipeline steps, expanded records) | +| `--logs` | `info`, `debug`, `trace` | (none) | Enable diagnostic logging to stderr | ### Output format selection -When `--output` / `-o` isn't given, mdvs picks a format using this priority chain: +When `--output` / `-o` isn't given, mdvs picks a format using this priority +chain: 1. **CLI flag** — `--output pretty|markdown|json` always wins when set. -2. **`default_output_format` in `mdvs.toml`** — a project-level override at the top of the file (`default_output_format = "markdown"`). +2. **`default_output_format` in `mdvs.toml`** — a project-level override at the + top of the file (`default_output_format = "markdown"`). 3. **Hard fallback** — `pretty`. -Same command → same output. The default does not depend on whether stdout is a terminal, a pipe, or a captured handle. Projects that want a different default (e.g. agent-curated KBs that prefer markdown on every invocation) set `default_output_format` in `mdvs.toml`. +Same command → same output. The default does not depend on whether stdout is a +terminal, a pipe, or a captured handle. Projects that want a different default +(e.g. agent-curated KBs that prefer markdown on every invocation) set +`default_output_format` in `mdvs.toml`. The three formats target different consumers: -- **`pretty`** — box-drawing tables for interactive terminal use. Adapts to terminal width. -- **`markdown`** — GFM pipe tables and `##` section headers. Use this when piping into docs, pasting into a PR description or issue, or when an LLM agent is reading mdvs output into its context — Markdown is the most token-efficient format that LLMs parse fluently. -- **`json`** — structured JSON for `jq` pipelines or programmatic consumers that want a strict contract. +- **`pretty`** — box-drawing tables for interactive terminal use. Adapts to + terminal width. +- **`markdown`** — GFM pipe tables and `##` section headers. Use this when + piping into docs, pasting into a PR description or issue, or when an LLM agent + is reading mdvs output into its context — Markdown is the most token-efficient + format that LLMs parse fluently. +- **`json`** — structured JSON for `jq` pipelines or programmatic consumers that + want a strict contract. --- @@ -49,7 +63,8 @@ The three formats target different consumers: ### `default_output_format` -Optional. Overrides the hard `pretty` default for this project. Values: `"pretty"`, `"markdown"`, `"json"`. Always loses to an explicit `--output` flag. +Optional. Overrides the hard `pretty` default for this project. Values: +`"pretty"`, `"markdown"`, `"json"`. Always loses to an explicit `--output` flag. ```toml default_output_format = "markdown" @@ -58,7 +73,9 @@ default_output_format = "markdown" # ... ``` -Useful for vaults where the same default makes sense for every contributor — for example, an agent-curated KB that should produce Markdown for the agent's context on every invocation without anyone having to remember the flag. +Useful for vaults where the same default makes sense for every contributor — for +example, an agent-curated KB that should produce Markdown for the agent's +context on every invocation without anyone having to remember the flag. --- @@ -74,44 +91,57 @@ skip_gitignore = false frontmatter_format = "auto" ``` -| Field | Type | Default | Description | -|---|---|---|---| -| `glob` | String | `"**"` | Glob pattern for matching markdown files | -| `include_bare_files` | Boolean | `true` | Include files without frontmatter | -| `skip_gitignore` | Boolean | `false` | Don't read `.gitignore` patterns during scan | -| `frontmatter_format` | String | `"auto"` | Which frontmatter format(s) to accept — see [Frontmatter format](#frontmatter-format) | +| Field | Type | Default | Description | +| -------------------- | ------- | -------- | ------------------------------------------------------------------------------------- | +| `glob` | String | `"**"` | Glob pattern for matching markdown files | +| `include_bare_files` | Boolean | `true` | Include files without frontmatter | +| `skip_gitignore` | Boolean | `false` | Don't read `.gitignore` patterns during scan | +| `frontmatter_format` | String | `"auto"` | Which frontmatter format(s) to accept — see [Frontmatter format](#frontmatter-format) | -When `include_bare_files` is `true`, files without frontmatter participate in inference (empty field set) and validation (can trigger `MissingRequired`). When `false`, they're excluded from the scan entirely. +When `include_bare_files` is `true`, files without frontmatter participate in +inference (empty field set) and validation (can trigger `MissingRequired`). When +`false`, they're excluded from the scan entirely. ### Frontmatter format -mdvs accepts YAML, TOML, and JSON frontmatter. The `frontmatter_format` field takes one of four values: +mdvs accepts YAML, TOML, and JSON frontmatter. The `frontmatter_format` field +takes one of four values: -| Value | Behavior | -|---|---| -| `"auto"` (default) | Detect per file from the opening delimiter. See the probe table below. | -| `"yaml"` | Parse every file as YAML; reject `+++` or `{`-opened files with a clear error. | -| `"toml"` | Parse every file as TOML; reject `---` or `{`-opened files. | -| `"json"` | Parse every file as JSON; reject `---` or `+++`-opened files. | +| Value | Behavior | +| ------------------ | ------------------------------------------------------------------------------ | +| `"auto"` (default) | Detect per file from the opening delimiter. See the probe table below. | +| `"yaml"` | Parse every file as YAML; reject `+++` or `{`-opened files with a clear error. | +| `"toml"` | Parse every file as TOML; reject `---` or `{`-opened files. | +| `"json"` | Parse every file as JSON; reject `---` or `+++`-opened files. | -In **auto mode** (the default), mdvs reads the first non-empty line of each file to pick the engine: +In **auto mode** (the default), mdvs reads the first non-empty line of each file +to pick the engine: -| First non-empty line of a file | Format used | -|---|---| -| `---` | YAML | -| `+++` | TOML | -| starts with `{` | JSON (Hugo convention — the braces are part of the JSON object) | -| anything else | treated as a bare file (no frontmatter) | +| First non-empty line of a file | Format used | +| ------------------------------ | --------------------------------------------------------------- | +| `---` | YAML | +| `+++` | TOML | +| starts with `{` | JSON (Hugo convention — the braces are part of the JSON object) | +| anything else | treated as a bare file (no frontmatter) | The probe is one line per file. A single vault can mix all three formats freely. -The **forced modes** (`"yaml"` / `"toml"` / `"json"`) skip the probe and assume every scanned file uses that format. Files whose actual leading delimiter belongs to a different format produce a `FrontmatterUnrepresentable` error naming both the configured and detected formats. This is useful for opinionated repos (e.g., a Hugo site committed to TOML that wants `mdvs check` to fail loudly if someone slips in a `---` file). +The **forced modes** (`"yaml"` / `"toml"` / `"json"`) skip the probe and assume +every scanned file uses that format. Files whose actual leading delimiter +belongs to a different format produce a `FrontmatterUnrepresentable` error +naming both the configured and detected formats. This is useful for opinionated +repos (e.g., a Hugo site committed to TOML that wants `mdvs check` to fail +loudly if someone slips in a `---` file). -**Naming note.** `frontmatter_format = "toml"` controls how mdvs parses *frontmatter in `.md` files*. It has nothing to do with `mdvs.toml` itself — `mdvs.toml` is always TOML because it's a config file. Two unrelated uses of "TOML" in the project. +**Naming note.** `frontmatter_format = "toml"` controls how mdvs parses +_frontmatter in `.md` files_. It has nothing to do with `mdvs.toml` itself — +`mdvs.toml` is always TOML because it's a config file. Two unrelated uses of +"TOML" in the project. ## `[update]` -Placeholder for future update-specific settings. Currently empty — this section is hidden from `mdvs.toml` by default. +Placeholder for future update-specific settings. Currently empty — this section +is hidden from `mdvs.toml` by default. ## `[check]` @@ -122,15 +152,21 @@ Check command settings. auto_update = true ``` -| Field | Type | Default | Description | -|---|---|---|---| +| Field | Type | Default | Description | +| ------------- | ------- | -------------------------- | --------------------------------- | | `auto_update` | Boolean | `true` (written by `init`) | Auto-run update before validating | -When `auto_update` is `true`, `check` runs the update pipeline (scan, infer, write config) before validating. `init` writes `true` so interactive runs pick up new fields automatically. Set to `false` or pass `--no-update` for **deterministic CI validation** against the committed `mdvs.toml` — the only reason to opt out. The chain is cheap on unchanged corpora, so there's no performance argument either way for local use. +When `auto_update` is `true`, `check` runs the update pipeline (scan, infer, +write config) before validating. `init` writes `true` so interactive runs pick +up new fields automatically. Set to `false` or pass `--no-update` for +**deterministic CI validation** against the committed `mdvs.toml` — the only +reason to opt out. The chain is cheap on unchanged corpora, so there's no +performance argument either way for local use. ## `[embedding_model]` -Specifies the embedding model for semantic search. See [Embedding](./concepts/search.md#embedding) for available models. +Specifies the embedding model for semantic search. See +[Embedding](./concepts/search.md#embedding) for available models. ```toml [embedding_model] @@ -138,15 +174,18 @@ provider = "model2vec" name = "minishlab/potion-multilingual-128M" ``` -| Field | Type | Default | Description | -|---|---|---|---| -| `provider` | String | `"model2vec"` | Embedding provider (currently only `"model2vec"`) | -| `name` | String | `"minishlab/potion-multilingual-128M"` | HuggingFace model ID | -| `revision` | String | (none) | Pin to a specific HuggingFace commit SHA for reproducibility | +| Field | Type | Default | Description | +| ---------- | ------ | -------------------------------------- | ------------------------------------------------------------ | +| `provider` | String | `"model2vec"` | Embedding provider (currently only `"model2vec"`) | +| `name` | String | `"minishlab/potion-multilingual-128M"` | HuggingFace model ID | +| `revision` | String | (none) | Pin to a specific HuggingFace commit SHA for reproducibility | -The `provider` field can be omitted — it defaults to `"model2vec"`. The `revision` field only appears when explicitly set (e.g., via `build --set-revision`). +The `provider` field can be omitted — it defaults to `"model2vec"`. The +`revision` field only appears when explicitly set (e.g., via +`build --set-revision`). -Changing the model or revision after a build requires `build --force` to re-embed all files. +Changing the model or revision after a build requires `build --force` to +re-embed all files. ## `[chunking]` @@ -157,11 +196,13 @@ Controls semantic text splitting before embedding. max_chunk_size = 1024 ``` -| Field | Type | Default | Description | -|---|---|---|---| -| `max_chunk_size` | Integer | `1024` | Maximum chunk size in characters | +| Field | Type | Default | Description | +| ---------------- | ------- | ------- | -------------------------------- | +| `max_chunk_size` | Integer | `1024` | Maximum chunk size in characters | -The text splitter breaks each file's body into semantic chunks respecting markdown structure (headings, paragraphs, lists). Changing the chunk size after a build requires `build --force`. +The text splitter breaks each file's body into semantic chunks respecting +markdown structure (headings, paragraphs, lists). Changing the chunk size after +a build requires `build --force`. ## `[build]` @@ -172,44 +213,56 @@ Build workflow settings. auto_update = true ``` -| Field | Type | Default | Description | -|---|---|---|---| +| Field | Type | Default | Description | +| ------------- | ------- | -------------------------- | ------------------------------- | | `auto_update` | Boolean | `true` (written by `init`) | Auto-run update before building | -When `auto_update` is `true`, `build` runs the update pipeline before building. Set to `false` or pass `--no-update` for **deterministic CI builds** against the committed `mdvs.toml` — the only reason to opt out. The chain is cheap on unchanged corpora (no model load, no Lance write when nothing changed). +When `auto_update` is `true`, `build` runs the update pipeline before building. +Set to `false` or pass `--no-update` for **deterministic CI builds** against the +committed `mdvs.toml` — the only reason to opt out. The chain is cheap on +unchanged corpora (no model load, no Lance write when nothing changed). ## `[search]` -Settings for the [search](./commands/search.md) command, including how internal columns are named in `--where` queries. +Settings for the [search](./commands/search.md) command, including how internal +columns are named in `--where` queries. ```toml [search] default_limit = 10 ``` -| Field | Type | Default | Description | -|---|---|---|---| -| `default_limit` | Integer | `10` | Maximum results when `--limit` is not specified | -| `internal_prefix` | String | `""` | Prefix for internal column names in `--where` queries | -| `aliases` | Map | `{}` | Per-column name overrides for internal columns | -| `auto_update` | Boolean | `true` (written by `init`) | Auto-run update before building (when `auto_build` is true) | -| `auto_build` | Boolean | `true` (written by `init`) | Auto-run build before searching | - -The two `[search]` auto-flags are what makes a bare `mdvs search "query"` a one-shot operation — it'll re-infer, validate, embed, and query in a single command. Set them to `false` (or use `--no-update` / `--no-build` flags) for **deterministic CI search** against an already-built index, or in airgapped environments where the embedding model can't be re-downloaded. Locally there's no performance argument: the auto chain is a no-op when nothing changed. +| Field | Type | Default | Description | +| ----------------- | ------- | -------------------------- | ----------------------------------------------------------- | +| `default_limit` | Integer | `10` | Maximum results when `--limit` is not specified | +| `internal_prefix` | String | `""` | Prefix for internal column names in `--where` queries | +| `aliases` | Map | `{}` | Per-column name overrides for internal columns | +| `auto_update` | Boolean | `true` (written by `init`) | Auto-run update before building (when `auto_build` is true) | +| `auto_build` | Boolean | `true` (written by `init`) | Auto-run build before searching | + +The two `[search]` auto-flags are what makes a bare `mdvs search "query"` a +one-shot operation — it'll re-infer, validate, embed, and query in a single +command. Set them to `false` (or use `--no-update` / `--no-build` flags) for +**deterministic CI search** against an already-built index, or in airgapped +environments where the embedding model can't be re-downloaded. Locally there's +no performance argument: the auto chain is a no-op when nothing changed. ### Internal column names -Beyond your frontmatter fields, the search index stores bookkeeping columns that mdvs uses internally. These *internal columns* are available in `--where` queries: +Beyond your frontmatter fields, the search index stores bookkeeping columns that +mdvs uses internally. These _internal columns_ are available in `--where` +queries: -| Column | Contains | -|---|---| -| `filepath` | Relative file path (e.g., `blog/post.md`) | -| `file_id` | Unique identifier for each file | -| `chunk_text` | The plain-text body of each chunk (useful for `--where "chunk_text LIKE '%foo%'"`) | -| `content_hash` | Hash of the file body | -| `built_at` | Timestamp of last build | +| Column | Contains | +| -------------- | ---------------------------------------------------------------------------------- | +| `filepath` | Relative file path (e.g., `blog/post.md`) | +| `file_id` | Unique identifier for each file | +| `chunk_text` | The plain-text body of each chunk (useful for `--where "chunk_text LIKE '%foo%'"`) | +| `content_hash` | Hash of the file body | +| `built_at` | Timestamp of last build | -(Other columns — `chunk_id`, `chunk_index`, `start_line`, `end_line`, `embedding` — exist too but are rarely useful in `--where`.) +(Other columns — `chunk_id`, `chunk_index`, `start_line`, `end_line`, +`embedding` — exist too but are rarely useful in `--where`.) By default, these are referenced by their raw names: @@ -217,29 +270,42 @@ By default, these are referenced by their raw names: --where "filepath LIKE 'blog/%'" ``` -If a frontmatter field name collides with an internal column name (e.g., you have a field called `filepath`), the search command will error and suggest resolutions: +If a frontmatter field name collides with an internal column name (e.g., you +have a field called `filepath`), the search command will error and suggest +resolutions: + +1. **Set a prefix** so internal columns are addressed with a leading marker in + `--where`: -1. **Set a prefix** so internal columns are addressed with a leading marker in `--where`: ```toml [search] internal_prefix = "_" ``` - Now `_filepath`, `_file_id`, etc. refer to the internal columns in `--where` clauses, leaving the bare `filepath` free to mean your frontmatter field. (The on-disk column names don't change — only how the `--where` translator interprets them.) + + Now `_filepath`, `_file_id`, etc. refer to the internal columns in `--where` + clauses, leaving the bare `filepath` free to mean your frontmatter field. + (The on-disk column names don't change — only how the `--where` translator + interprets them.) 2. **Set a per-column alias** to rename just the colliding column in `--where`: + ```toml [search.aliases] filepath = "path" ``` - Now `path` refers to the internal `filepath` column, and bare `filepath` refers to your frontmatter field. + + Now `path` refers to the internal `filepath` column, and bare `filepath` + refers to your frontmatter field. 3. **Rename the frontmatter field** in your markdown files. -Aliases take precedence over the prefix. See the [Search Guide](./search-guide.md) for full `--where` reference. +Aliases take precedence over the prefix. See the +[Search Guide](./search-guide.md) for full `--where` reference. ## `[fields]` -Defines field constraints and the ignore list. This is the largest section — it contains one `[[fields.field]]` entry per constrained field. +Defines field constraints and the ignore list. This is the largest section — it +contains one `[[fields.field]]` entry per constrained field. ### Ignore list @@ -248,7 +314,10 @@ Defines field constraints and the ignore list. This is the largest section — i ignore = ["internal_id", "temp_notes"] ``` -Fields in the `ignore` list are known but unconstrained — they skip all validation and are not reported as new fields by [check](./commands/check.md) or [update](./commands/update.md). A field cannot be in both `ignore` and `[[fields.field]]`. +Fields in the `ignore` list are known but unconstrained — they skip all +validation and are not reported as new fields by [check](./commands/check.md) or +[update](./commands/update.md). A field cannot be in both `ignore` and +`[[fields.field]]`. ### Field definitions @@ -263,17 +332,18 @@ required = ["blog/**", "projects/**"] nullable = false ``` -| Field | Type | Default | Description | -|---|---|---|---| -| `name` | String | (required) | Frontmatter key | -| `type` | FieldType | `"String"` | Expected value type | -| `allowed` | Array(String) | `["**"]` | Glob patterns where the field may appear | -| `required` | Array(String) | `[]` | Glob patterns where the field must be present | -| `nullable` | Boolean | `true` | Whether null values are accepted | -| `constraints` | Table | (absent) | Optional value constraints (see [Constraints](#constraints)) | -| `preprocess` | Array(String) | `[]` | Stage 2 value preprocessors — see [Preprocessors](#preprocessors) | +| Field | Type | Default | Description | +| ------------- | ------------- | ---------- | ----------------------------------------------------------------- | +| `name` | String | (required) | Frontmatter key | +| `type` | FieldType | `"String"` | Expected value type | +| `allowed` | Array(String) | `["**"]` | Glob patterns where the field may appear | +| `required` | Array(String) | `[]` | Glob patterns where the field must be present | +| `nullable` | Boolean | `true` | Whether null values are accepted | +| `constraints` | Table | (absent) | Optional value constraints (see [Constraints](#constraints)) | +| `preprocess` | Array(String) | `[]` | Stage 2 value preprocessors — see [Preprocessors](#preprocessors) | -All fields except `name` have permissive defaults. A minimal entry with just a name: +All fields except `name` have permissive defaults. A minimal entry with just a +name: ```toml [[fields.field]] @@ -291,7 +361,11 @@ required = [] nullable = true ``` -This is not the same as putting the field in the `ignore` list. Both prevent the field from being reported as new during `update`, but a `[[fields.field]]` entry tracks the field — it appears in `info` output with its type and patterns, and can be targeted by `update reinfer`. The `ignore` list simply silences the field: no validation, no detail in `info`. +This is not the same as putting the field in the `ignore` list. Both prevent the +field from being reported as new during `update`, but a `[[fields.field]]` entry +tracks the field — it appears in `info` output with its type and patterns, and +can be targeted by `update reinfer`. The `ignore` list simply silences the +field: no validation, no detail in `info`. ### Type syntax @@ -301,7 +375,10 @@ Scalar types are plain strings: type = "String" # also: "Boolean", "Integer", "Float", "Date", "DateTime" ``` -`Date` and `DateTime` accept RFC 3339 values only (`YYYY-MM-DD` for Date, `YYYY-MM-DDTHH:MM:SS[.frac]` for DateTime). See [Date and DateTime](./concepts/types.md#date-and-datetime) for the exact accepted shapes and storage semantics. +`Date` and `DateTime` accept RFC 3339 values only (`YYYY-MM-DD` for Date, +`YYYY-MM-DDTHH:MM:SS[.frac]` for DateTime). See +[Date and DateTime](./concepts/types.md#date-and-datetime) for the exact +accepted shapes and storage semantics. Arrays use a function-style string: @@ -309,7 +386,11 @@ Arrays use a function-style string: type = "Array(String)" ``` -**Structured types are not supported on disk.** Nested Objects in frontmatter are expressed via dotted-name leaf fields — see [Types](./concepts/types.md) for the flattening rule. Arrays of structured items (`Array(Object{...})`) have no first-class representation in v0; use **parallel scalar arrays** as a workaround: +**Structured types are not supported on disk.** Nested Objects in frontmatter +are expressed via dotted-name leaf fields — see [Types](./concepts/types.md) for +the flattening rule. Arrays of structured items (`Array(Object{...})`) have no +first-class representation in v0; use **parallel scalar arrays** as a +workaround: ```toml # Instead of an unsupported Array(Object{timestamp, value}): @@ -329,28 +410,38 @@ Type := Scalar | Array(Scalar) Scalar := String | Integer | Float | Boolean | Date | DateTime ``` -See [Types](./concepts/types.md) for the full type system, including widening rules. +See [Types](./concepts/types.md) for the full type system, including widening +rules. ### Path patterns -`allowed` and `required` are lists of glob patterns matched against relative file paths: +`allowed` and `required` are lists of glob patterns matched against relative +file paths: ```toml allowed = ["blog/**", "projects/alpha/**"] required = ["blog/published/**"] ``` -Patterns must end with `/*` (direct children) or `/**` (full subtree), or be exactly `*` or `**`. Bare paths like `blog` or file names like `blog/post.md` are not valid. +Patterns must end with `/*` (direct children) or `/**` (full subtree), or be +exactly `*` or `**`. Bare paths like `blog` or file names like `blog/post.md` +are not valid. -The invariant `required ⊆ allowed` is enforced — every required glob must be covered by some allowed glob. For example, `allowed = ["meetings/**"]` covers `required = ["meetings/all-hands/**"]` because any path matching the required pattern also matches the allowed one. +The invariant `required ⊆ allowed` is enforced — every required glob must be +covered by some allowed glob. For example, `allowed = ["meetings/**"]` covers +`required = ["meetings/all-hands/**"]` because any path matching the required +pattern also matches the allowed one. -See [Schema Inference](./concepts/schema.md#path-patterns) for how these patterns are computed. +See [Schema Inference](./concepts/schema.md#path-patterns) for how these +patterns are computed. ### Constraints -The optional `[fields.field.constraints]` sub-table adds value constraints beyond type checking. +The optional `[fields.field.constraints]` sub-table adds value constraints +beyond type checking. -**`categories`** — restricts values to an enumerated set (String, Integer, or arrays of either): +**`categories`** — restricts values to an enumerated set (String, Integer, or +arrays of either): ```toml [[fields.field]] @@ -361,7 +452,8 @@ type = "String" categories = ["active", "archived", "completed", "draft", "published"] ``` -**`min` / `max`** — restricts numeric values to an inclusive range (Integer, Float, or arrays of either). Both bounds are optional: +**`min` / `max`** — restricts numeric values to an inclusive range (Integer, +Float, or arrays of either). Both bounds are optional: ```toml [[fields.field]] @@ -373,7 +465,8 @@ min = 1 max = 5 ``` -**`min_length` / `max_length`** — bounds string length (Unicode scalar count) or array length: +**`min_length` / `max_length`** — bounds string length (Unicode scalar count) or +array length: ```toml [[fields.field]] @@ -396,16 +489,21 @@ type = "String" pattern = '^v\d+\.\d+\.\d+$' ``` -Categories are auto-inferred during `init` and `update reinfer`. Range constraints are not auto-inferred but can be inferred on demand with `update reinfer --with=range`. Length and pattern are not auto-inferred — add them by hand. See [Constraints](./concepts/constraints.md) for the full reference. +Categories are auto-inferred during `init` and `update reinfer`. Range +constraints are not auto-inferred but can be inferred on demand with +`update reinfer --with=range`. Length and pattern are not auto-inferred +— add them by hand. See [Constraints](./concepts/constraints.md) for the full +reference. ### Preprocessors -The optional `preprocess` array on a field declares value transformations that run **before** validation. Two built-in stages: +The optional `preprocess` array on a field declares value transformations that +run **before** validation. Two built-in stages: -| Stage | Applies to | Effect | -|---|---|---| -| `coerce-to-string` | `String`, `Array(String)` | Serialize non-string JSON values to their JSON string form before validation | -| `widen-int-to-float` | `Float`, `Array(Float)` | Treat integer values as their float equivalent | +| Stage | Applies to | Effect | +| -------------------- | ------------------------- | ---------------------------------------------------------------------------- | +| `coerce-to-string` | `String`, `Array(String)` | Serialize non-string JSON values to their JSON string form before validation | +| `widen-int-to-float` | `Float`, `Array(Float)` | Treat integer values as their float equivalent | ```toml [[fields.field]] @@ -414,9 +512,14 @@ type = "String" preprocess = ["coerce-to-string"] ``` -Preprocessors are **auto-inferred** during `init` and `update reinfer` based on observed type-widening events: a field that widened to `String` because of mixed-type observations gets `coerce-to-string`; a `Float` field that observed integers gets `widen-int-to-float`. An empty `preprocess` array means strict validation — no coercion. +Preprocessors are **auto-inferred** during `init` and `update reinfer` based on +observed type-widening events: a field that widened to `String` because of +mixed-type observations gets `coerce-to-string`; a `Float` field that observed +integers gets `widen-int-to-float`. An empty `preprocess` array means strict +validation — no coercion. -Each entry must be applicable to the field's type, and duplicates are rejected at config load. See [Types & Widening](./concepts/types.md) for the full rules. +Each entry must be applicable to the field's type, and duplicates are rejected +at config load. See [Types & Widening](./concepts/types.md) for the full rules. ### Inference thresholds @@ -428,12 +531,13 @@ max_categories = 10 min_category_repetition = 3 ``` -| Field | Type | Default | Description | -|---|---|---|---| -| `max_categories` | Integer | `10` | Max distinct values for a field to be inferred as categorical | -| `min_category_repetition` | Integer | `3` | Min average repetition (occurrences / distinct) for categorical inference | +| Field | Type | Default | Description | +| ------------------------- | ------- | ------- | ------------------------------------------------------------------------- | +| `max_categories` | Integer | `10` | Max distinct values for a field to be inferred as categorical | +| `min_category_repetition` | Integer | `3` | Min average repetition (occurrences / distinct) for categorical inference | -These are hidden from `mdvs.toml` when set to their defaults. They only affect auto-inference — manually written `categories` are unaffected. +These are hidden from `mdvs.toml` when set to their defaults. They only affect +auto-inference — manually written `categories` are unaffected. ## Example diff --git a/book/src/getting-started.md b/book/src/getting-started.md index 014e976..b92c239 100644 --- a/book/src/getting-started.md +++ b/book/src/getting-started.md @@ -1,6 +1,7 @@ # Getting Started -Install mdvs, run it on a real directory, and search your first query — all in under five minutes. +Install mdvs, run it on a real directory, and search your first query — all in +under five minutes. ## Install @@ -8,11 +9,14 @@ Install mdvs, run it on a real directory, and search your first query — all in cargo install mdvs ``` -You need a working [Rust toolchain](https://rustup.rs/). Prebuilt binaries will be available once the crate is published. +You need a working [Rust toolchain](https://rustup.rs/). Prebuilt binaries will +be available once the crate is published. ## Get the example files -This book uses a fixture called `example_kb` — a fictional research lab's knowledge base with ~46 markdown files, varied frontmatter, and a few deliberate inconsistencies. Clone the repo to follow along: +This book uses a fixture called `example_kb` — a fictional research lab's +knowledge base with ~46 markdown files, varied frontmatter, and a few deliberate +inconsistencies. Clone the repo to follow along: ```bash git clone https://github.com/edochi/mdvs.git @@ -27,7 +31,8 @@ Run `mdvs init` on the example directory: mdvs init example_kb ``` -mdvs scans every markdown file, extracts frontmatter, and infers a typed schema. Each discovered field is shown as its own key-value table: +mdvs scans every markdown file, extracts frontmatter, and infers a typed schema. +Each discovered field is shown as its own key-value table: ``` Initialized 43 files — 37 field(s) @@ -86,10 +91,14 @@ Initialized mdvs in 'example_kb' That command did three things: 1. **Scanned** 43 markdown files and extracted their YAML frontmatter -2. **Inferred** 37 typed fields — strings, integers, floats, booleans, arrays, even a nested object (`calibration`) +2. **Inferred** 37 typed fields — strings, integers, floats, booleans, arrays, + even a nested object (`calibration`) 3. **Wrote** `mdvs.toml` with the inferred schema -Notice the `files` row: `draft` appears in 8 out of 43 files — all in `blog/`. `sensor_type` in 3 out of 43 — all in `projects/alpha/notes/`. mdvs captured not just the types, but *where* each field belongs, via the `required` and `allowed` glob patterns. +Notice the `files` row: `draft` appears in 8 out of 43 files — all in `blog/`. +`sensor_type` in 3 out of 43 — all in `projects/alpha/notes/`. mdvs captured not +just the types, but _where_ each field belongs, via the `required` and `allowed` +glob patterns. Here's what a field definition looks like in `mdvs.toml`: @@ -102,9 +111,13 @@ required = ["projects/alpha/notes/**"] nullable = false ``` -This means `sensor_type` is allowed only in experiment notes, and required there. If it appears in a blog post, `check` will flag it. If it's missing from an experiment note, `check` will flag that too. +This means `sensor_type` is allowed only in experiment notes, and required +there. If it appears in a blog post, `check` will flag it. If it's missing from +an experiment note, `check` will flag that too. -One artifact is created by `init`: **`mdvs.toml`** — the schema file. Commit this to version control. The `.mdvs/` directory (search index) is created later on first `build` or `search`. +One artifact is created by `init`: **`mdvs.toml`** — the schema file. Commit +this to version control. The `.mdvs/` directory (search index) is created later +on first `build` or `search`. ## Validate @@ -118,16 +131,21 @@ mdvs check example_kb Checked 43 files — no violations ``` -Since `mdvs init` just inferred the schema from these same files, everything passes. The power of `check` comes after you tighten the schema — or when files drift from it. Try adding `sensor_type: SPR-A1` to a blog post — mdvs will flag it as `Disallowed` because that field doesn't belong there. +Since `mdvs init` just inferred the schema from these same files, everything +passes. The power of `check` comes after you tighten the schema — or when files +drift from it. Try adding `sensor_type: SPR-A1` to a blog post — mdvs will flag +it as `Disallowed` because that field doesn't belong there. ### What violations look like Open `mdvs.toml` and make a few changes to tighten the constraints: - Require `observation_notes` in all experiment files (currently optional) -- Change `convergence_ms` type from `Integer` to `Boolean` (simulating a type mismatch) +- Change `convergence_ms` type from `Integer` to `Boolean` (simulating a type + mismatch) - Set `drift_rate` to non-nullable (one file has `drift_rate: null`) -- Restrict `firmware_version` to only appear in `people/interns/**` (it currently appears in `people/*`) +- Restrict `firmware_version` to only appear in `people/interns/**` (it + currently appears in `people/*`) Run `check` again: @@ -176,22 +194,28 @@ Violations (4): Four violation types, each catching a different kind of problem: -| Violation | Meaning | -|---|---| -| `Missing required` | A file in a required path is missing the field | -| `Wrong type` | The value doesn't match the declared type | +| Violation | Meaning | +| ------------------------ | ---------------------------------------------------------- | +| `Missing required` | A file in a required path is missing the field | +| `Wrong type` | The value doesn't match the declared type | | `Null value not allowed` | The field is present but `null`, and `nullable` is `false` | -| `Not allowed` | The field appears in a file outside its `allowed` paths | +| `Not allowed` | The field appears in a file outside its `allowed` paths | -Each violation table shows the field name, the kind of violation, the violated rule, and the affected files. See [check](./commands/check.md) for the full reference. +Each violation table shows the field name, the kind of violation, the violated +rule, and the affected files. See [check](./commands/check.md) for the full +reference. -Revert your changes to `mdvs.toml` before continuing (or re-run `mdvs init example_kb --force` to regenerate it). +Revert your changes to `mdvs.toml` before continuing (or re-run +`mdvs init example_kb --force` to regenerate it). ## Search -Query the index with natural language. On first run, `search` auto-builds the index: +Query the index with natural language. On first run, `search` auto-builds the +index: -> **Note:** The first `search` or `build` downloads the embedding model from HuggingFace (~30 MB for the default model). This is a one-time download — subsequent runs use the cached model and start instantly. +> **Note:** The first `search` or `build` downloads the embedding model from +> HuggingFace (~30 MB for the default model). This is a one-time download — +> subsequent runs use the cached model and start instantly. ```bash mdvs search "calibration" example_kb @@ -227,7 +251,12 @@ Searched "calibration" — 10 hits ... ``` -By default `mdvs search` runs in **hybrid mode** — it combines a semantic (vector) match with a full-text (BM25) match and reranks the results, so a typo-friendly natural-language query and an exact-keyword query both work. The `score` is a relevance score from the reranker (higher is better). Pass `--mode semantic` or `--mode fulltext` to use one signal alone. The `text` row shows the best-matching chunk from each file. +By default `mdvs search` runs in **hybrid mode** — it combines a semantic +(vector) match with a full-text (BM25) match and reranks the results, so a +typo-friendly natural-language query and an exact-keyword query both work. The +`score` is a relevance score from the reranker (higher is better). Pass +`--mode semantic` or `--mode fulltext` to use one signal alone. The `text` row +shows the best-matching chunk from each file. ### Filtering with `--where` @@ -257,11 +286,17 @@ Searched "quantum" — 3 hits ... ``` -Only files with `status: active` in their frontmatter are included. The `--where` clause supports any SQL expression — boolean logic, comparisons, array functions, and more. See the [Search Guide](./search-guide.md) for the full syntax. +Only files with `status: active` in their frontmatter are included. The +`--where` clause supports any SQL expression — boolean logic, comparisons, array +functions, and more. See the [Search Guide](./search-guide.md) for the full +syntax. ## What's next -- **[Concepts](./concepts.md)** — How schema inference, types, and validation work under the hood +- **[Concepts](./concepts.md)** — How schema inference, types, and validation + work under the hood - **[Commands](./commands/init.md)** — Full reference for every command and flag -- **[Configuration](./configuration.md)** — Customize `mdvs.toml` to tighten your schema -- **[Search Guide](./search-guide.md)** — Complex queries: arrays, nested objects, combined filters +- **[Configuration](./configuration.md)** — Customize `mdvs.toml` to tighten + your schema +- **[Search Guide](./search-guide.md)** — Complex queries: arrays, nested + objects, combined filters diff --git a/book/src/introduction.md b/book/src/introduction.md index f31d2f1..4c86704 100644 --- a/book/src/introduction.md +++ b/book/src/introduction.md @@ -1,18 +1,24 @@ # Introduction -mdvs treats your markdown directory like a database. It scans your files, infers a typed schema from frontmatter, validates it, and builds a local search index — all in a single binary with no external services. +mdvs treats your markdown directory like a database. It scans your files, infers +a typed schema from frontmatter, validates it, and builds a local search index — +all in a single binary with no external services. -Not a document database. A database *for* documents. +Not a document database. A database _for_ documents. ## The challenge -Markdown directories grow organically. You start with a few notes, add frontmatter when it's useful, and eventually have hundreds of files with inconsistent metadata. Tags are misspelled. Required fields are missing. You can't find anything without `grep`. +Markdown directories grow organically. You start with a few notes, add +frontmatter when it's useful, and eventually have hundreds of files with +inconsistent metadata. Tags are misspelled. Required fields are missing. You +can't find anything without `grep`. mdvs gives you structure without forcing you to change how you write. ## Frontmatter -Frontmatter is the YAML block between `---` fences at the top of a markdown file. It stores structured metadata alongside your content: +Frontmatter is the YAML block between `---` fences at the top of a markdown +file. It stores structured metadata alongside your content: ```yaml --- @@ -30,58 +36,89 @@ tags: # Array(String) # Your markdown content starts here... ``` -mdvs recognizes these types automatically. When it scans your files, it infers the type of each field from the values it finds — no configuration needed. +mdvs recognizes these types automatically. When it scans your files, it infers +the type of each field from the values it finds — no configuration needed. -> TOML (`+++`) and JSON (`{...}`) frontmatter are also supported, auto-detected per file. This guide uses YAML throughout; see [`[scan].frontmatter_format`](./configuration.md#frontmatter-format) for the format knob and the [Hugo recipe](./recipes/hugo.md) for mixed-format vaults. +> TOML (`+++`) and JSON (`{...}`) frontmatter are also supported, auto-detected +> per file. This guide uses YAML throughout; see +> [`[scan].frontmatter_format`](./configuration.md#frontmatter-format) for the +> format knob and the [Hugo recipe](./recipes/hugo.md) for mixed-format vaults. ## Directory-aware schema mdvs infers a three-dimensional schema from your files: -- **Types** — boolean, integer, float, string, arrays, nested objects. Inferred automatically, with widening when files disagree. -- **Paths** — which fields belong in which directories. `draft` only in `blog/`, `sensor_type` only in `projects/alpha/notes/`. Captured as `allowed` and `required` glob patterns. +- **Types** — boolean, integer, float, string, arrays, nested objects. Inferred + automatically, with widening when files disagree. +- **Paths** — which fields belong in which directories. `draft` only in `blog/`, + `sensor_type` only in `projects/alpha/notes/`. Captured as `allowed` and + `required` glob patterns. - **Nullability** — whether a field can be null. Tracked per field. -This means different directories can have different fields with different constraints — all inferred automatically from your existing files. +This means different directories can have different fields with different +constraints — all inferred automatically from your existing files. -> **Tightest fit:** `mdvs init` infers the strictest schema that's consistent with your existing files. A field is inferred as *allowed* in a directory if at least one file there has it. It's inferred as *required* if every file there has it. These rules propagate up — if every subdirectory requires a field, the parent directory does too. The result is the tightest set of constraints where `check` still returns zero violations. You can always loosen them later. +> **Tightest fit:** `mdvs init` infers the strictest schema that's consistent +> with your existing files. A field is inferred as _allowed_ in a directory if +> at least one file there has it. It's inferred as _required_ if every file +> there has it. These rules propagate up — if every subdirectory requires a +> field, the parent directory does too. The result is the tightest set of +> constraints where `check` still returns zero violations. You can always loosen +> them later. ## Two layers mdvs has two distinct capabilities that work independently: -**Validation** — Scan your files, infer what frontmatter fields exist, which directories they appear in, and what types they have. Write the result to `mdvs.toml`. Then validate files against that schema. No model, no index, nothing to download. +**Validation** — Scan your files, infer what frontmatter fields exist, which +directories they appear in, and what types they have. Write the result to +`mdvs.toml`. Then validate files against that schema. No model, no index, +nothing to download. -**Search** — Chunk your markdown, embed it with a lightweight local model, store the chunks and vectors in a Lance dataset under `.mdvs/`, and query with natural language. Choose semantic (vector), full-text (BM25), or hybrid (both, reranked) — and filter results on any frontmatter field using standard SQL. +**Search** — Chunk your markdown, embed it with a lightweight local model, store +the chunks and vectors in a Lance dataset under `.mdvs/`, and query with natural +language. Choose semantic (vector), full-text (BM25), or hybrid (both, reranked) +— and filter results on any frontmatter field using standard SQL. -You need validation without search? Run `mdvs init`, customize the fields in `mdvs.toml`, and run `mdvs check`. +You need validation without search? Run `mdvs init`, customize the fields in +`mdvs.toml`, and run `mdvs check`. -You want search without validation? Just run `mdvs init` and `mdvs search`. The inferred schema is used to extract metadata for search results, but you don't have to worry about it if you don't want to. +You want search without validation? Just run `mdvs init` and `mdvs search`. The +inferred schema is used to extract metadata for search results, but you don't +have to worry about it if you don't want to. -Use them together for the best experience, or separately if that's what you need. +Use them together for the best experience, or separately if that's what you +need. ## Using a nested directory of markdown files as a database -You can think of mdvs as a layer on top of your markdown files that gives you database-like capabilities. Here's a rough mapping of concepts and commands: +You can think of mdvs as a layer on top of your markdown files that gives you +database-like capabilities. Here's a rough mapping of concepts and commands: -| Concept | Database | mdvs | -|---|---|---| -| Define structure | `CREATE TABLE` | `mdvs init` | -| Per-table columns | Different columns per table | Per-directory fields via `allowed`/`required` globs | -| Enforce constraints | Constraint validation | `mdvs check` | -| Evolve structure | `ALTER TABLE` | `mdvs update` | -| Create an index | `CREATE INDEX` | `mdvs build` | -| Query | `SELECT ... WHERE ... ORDER BY` | `mdvs search --where` | +| Concept | Database | mdvs | +| ------------------- | ------------------------------- | --------------------------------------------------- | +| Define structure | `CREATE TABLE` | `mdvs init` | +| Per-table columns | Different columns per table | Per-directory fields via `allowed`/`required` globs | +| Enforce constraints | Constraint validation | `mdvs check` | +| Evolve structure | `ALTER TABLE` | `mdvs update` | +| Create an index | `CREATE INDEX` | `mdvs build` | +| Query | `SELECT ... WHERE ... ORDER BY` | `mdvs search --where` | -Two artifacts: `mdvs.toml` (your schema, to be committed) and `.mdvs/` (the search index, can be ignored by version control). +Two artifacts: `mdvs.toml` (your schema, to be committed) and `.mdvs/` (the +search index, can be ignored by version control). ## What this book covers -This book uses a fictional research lab knowledge base ([example_kb](https://github.com/edochi/mdvs/tree/main/example_kb)) as a running example. Every command, every output, every query is real and reproducible. +This book uses a fictional research lab knowledge base +([example_kb](https://github.com/edochi/mdvs/tree/main/example_kb)) as a running +example. Every command, every output, every query is real and reproducible. -- **[Getting Started](./getting-started.md)** — Install mdvs and run it on the example vault -- **[Concepts](./concepts.md)** — How schema inference, types, and validation work +- **[Getting Started](./getting-started.md)** — Install mdvs and run it on the + example vault +- **[Concepts](./concepts.md)** — How schema inference, types, and validation + work - **[Commands](./commands/init.md)** — Full reference for all 8 commands - **[Configuration](./configuration.md)** — The `mdvs.toml` file explained -- **[Search Guide](./search-guide.md)** — SQL filtering, array queries, and ranking +- **[Search Guide](./search-guide.md)** — SQL filtering, array queries, and + ranking - **[Recipes](./recipes/obsidian.md)** — Obsidian setup, CI integration diff --git a/book/src/recipes.md b/book/src/recipes.md index ac95e86..5a65e7d 100644 --- a/book/src/recipes.md +++ b/book/src/recipes.md @@ -1,8 +1,16 @@ # Recipes -Walkthroughs for pointing mdvs at common markdown ecosystems and the agents that work in them. +Walkthroughs for pointing mdvs at common markdown ecosystems and the agents that +work in them. -- **[Agent harnesses](./recipes/agent-harnesses.md)** — Wiring mdvs into Claude Code, Codex, Cursor, OpenCode, Antigravity. Skill file, project-rules snippet, validate-on-write hook, search-nudge hook. Per-platform pages for copy-paste install; overview page for the architecture and how to extend to a new harness. -- **[Obsidian](./recipes/obsidian.md)** — YAML-frontmatter vaults, `.mdvsignore` patterns, Dataview caveats, common validation setups -- **[Hugo](./recipes/hugo.md)** — Mixed-format sites (YAML / TOML / JSON), native TOML date queries, forced-format mode for opinionated repos -- **[CI](./recipes/ci.md)** — Running `mdvs check` in a pipeline as a frontmatter linter +- **[Agent harnesses](./recipes/agent-harnesses.md)** — Wiring mdvs into Claude + Code, Codex, Cursor, OpenCode, Antigravity. Skill file, project-rules snippet, + validate-on-write hook, search-nudge hook. Per-platform pages for copy-paste + install; overview page for the architecture and how to extend to a new + harness. +- **[Obsidian](./recipes/obsidian.md)** — YAML-frontmatter vaults, `.mdvsignore` + patterns, Dataview caveats, common validation setups +- **[Hugo](./recipes/hugo.md)** — Mixed-format sites (YAML / TOML / JSON), + native TOML date queries, forced-format mode for opinionated repos +- **[CI](./recipes/ci.md)** — Running `mdvs check` in a pipeline as a + frontmatter linter diff --git a/book/src/recipes/agent-harnesses.md b/book/src/recipes/agent-harnesses.md index ab50576..8185489 100644 --- a/book/src/recipes/agent-harnesses.md +++ b/book/src/recipes/agent-harnesses.md @@ -2,9 +2,12 @@ mdvs ships agent integration in three pieces: -1. A **skill** (the [Agent Skills standard](https://agentskills.io)) — works in any harness that loads `.md` skills. -2. A **project-rules snippet** — works in any harness that reads `AGENTS.md` / `CLAUDE.md` / `.cursor/rules`. -3. A **PostToolUse hook** that calls `mdvs hook handle` — only verified end-to-end on Claude Code today. +1. A **skill** (the [Agent Skills standard](https://agentskills.io)) — works in + any harness that loads `.md` skills. +2. A **project-rules snippet** — works in any harness that reads `AGENTS.md` / + `CLAUDE.md` / `.cursor/rules`. +3. A **PostToolUse hook** that calls `mdvs hook handle` — only verified + end-to-end on Claude Code today. Per-harness install steps in the left nav. @@ -12,28 +15,50 @@ Per-harness install steps in the left nav. When the agent edits a markdown file in your vault: -1. The harness's PostToolUse hook fires the configured `mdvs hook handle` command. -2. mdvs reads the tool-call payload, walks up to find `mdvs.toml`. If the edit happened outside any vault, the hook stays silent. -3. mdvs runs `check` on the vault. If the file is clean, the hook stays silent (no noise on the happy path). -4. If there are violations, mdvs writes a Claude-Code-shaped envelope JSON to stdout. The harness reads it and surfaces the markdown body to the agent through `additionalContext` and the pretty render to the user through `systemMessage`. -5. The agent sees the violation and reacts on its next turn — per the [schema-evolution loop](https://github.com/edochi/mdvs/blob/main/crates/mdvs/scaffolding/skill/SKILL.md): if it's a mistake, fix the file; if it's intentional (KB evolving), surface the deviation to the user and propose updating `mdvs.toml`. +1. The harness's PostToolUse hook fires the configured `mdvs hook handle` + command. +2. mdvs reads the tool-call payload, walks up to find `mdvs.toml`. If the edit + happened outside any vault, the hook stays silent. +3. mdvs runs `check` on the vault. If the file is clean, the hook stays silent + (no noise on the happy path). +4. If there are violations, mdvs writes a Claude-Code-shaped envelope JSON to + stdout. The harness reads it and surfaces the markdown body to the agent + through `additionalContext` and the pretty render to the user through + `systemMessage`. +5. The agent sees the violation and reacts on its next turn — per the + [schema-evolution loop](https://github.com/edochi/mdvs/blob/main/crates/mdvs/scaffolding/skill/SKILL.md): + if it's a mistake, fix the file; if it's intentional (KB evolving), surface + the deviation to the user and propose updating `mdvs.toml`. -A separate **search-nudge** hook fires after every Bash command that runs `grep` / `rg` / `find` / `ag` / `ack` / `fd` / `git grep`. If the agent's cwd is inside an mdvs vault, the hook surfaces a one-line tip suggesting `mdvs search`. Like validate, it's non-blocking — the agent decides whether to switch tools. +A separate **search-nudge** hook fires after every Bash command that runs `grep` +/ `rg` / `find` / `ag` / `ack` / `fd` / `git grep`. If the agent's cwd is inside +an mdvs vault, the hook surfaces a one-line tip suggesting `mdvs search`. Like +validate, it's non-blocking — the agent decides whether to switch tools. ## Per-platform support -| Platform | Skill | Snippet | Hooks | -|---|---|---|---| -| [Claude Code](agent-harnesses/claude-code.md) | ✓ | ✓ | ✓ | -| [Codex](agent-harnesses/codex.md) | ✓ | ✓ | see [Codex hooks docs](https://developers.openai.com/codex/hooks) | -| [Cursor](agent-harnesses/cursor.md) | ✓ | ✓ | see [Cursor hooks docs](https://cursor.com/docs/hooks) | -| [OpenCode](agent-harnesses/opencode.md) | ✓ | ✓ | see [OpenCode docs](https://opencode.ai/docs/) | -| [Antigravity](agent-harnesses/antigravity.md) | ✓ | ✓ | see [Gemini CLI hooks docs](https://github.com/google-gemini/gemini-cli/tree/main/docs/hooks) | +| Platform | Skill | Snippet | Hooks | +| --------------------------------------------- | ----- | ------- | --------------------------------------------------------------------------------------------- | +| [Claude Code](agent-harnesses/claude-code.md) | ✓ | ✓ | ✓ | +| [Codex](agent-harnesses/codex.md) | ✓ | ✓ | see [Codex hooks docs](https://developers.openai.com/codex/hooks) | +| [Cursor](agent-harnesses/cursor.md) | ✓ | ✓ | see [Cursor hooks docs](https://cursor.com/docs/hooks) | +| [OpenCode](agent-harnesses/opencode.md) | ✓ | ✓ | see [OpenCode docs](https://opencode.ai/docs/) | +| [Antigravity](agent-harnesses/antigravity.md) | ✓ | ✓ | see [Gemini CLI hooks docs](https://github.com/google-gemini/gemini-cli/tree/main/docs/hooks) | ## Pre-commit hook -A **pre-commit hook** is a script git runs locally before each `git commit` — if it exits non-zero, the commit is blocked. Running `mdvs check --no-update` there catches frontmatter violations before they reach the repo, **regardless of how the file was edited** — agent, IDE, or by hand. +A **pre-commit hook** is a script git runs locally before each `git commit` — if +it exits non-zero, the commit is blocked. Running `mdvs check --no-update` there +catches frontmatter violations before they reach the repo, **regardless of how +the file was edited** — agent, IDE, or by hand. -That makes it the harness-independent safety net, and the recommended fallback for harnesses where the PostToolUse hook isn't wired up. It complements the hooks above rather than replacing them: the PostToolUse hook tells the agent mid-session, while the pre-commit hook is the backstop that catches whatever slipped through. +That makes it the harness-independent safety net, and the recommended fallback +for harnesses where the PostToolUse hook isn't wired up. It complements the +hooks above rather than replacing them: the PostToolUse hook tells the agent +mid-session, while the pre-commit hook is the backstop that catches whatever +slipped through. -Full setup — both the [pre-commit framework](https://pre-commit.com/) and the plain `.git/hooks/pre-commit` script — is in the [pre-commit recipe](./pre-commit.md). For CI-side validation (catches violations even if a contributor skipped the local hook), see the [CI recipe](./ci.md). +Full setup — both the [pre-commit framework](https://pre-commit.com/) and the +plain `.git/hooks/pre-commit` script — is in the +[pre-commit recipe](./pre-commit.md). For CI-side validation (catches violations +even if a contributor skipped the local hook), see the [CI recipe](./ci.md). diff --git a/book/src/recipes/agent-harnesses/antigravity.md b/book/src/recipes/agent-harnesses/antigravity.md index 823bbeb..e83a302 100644 --- a/book/src/recipes/agent-harnesses/antigravity.md +++ b/book/src/recipes/agent-harnesses/antigravity.md @@ -8,23 +8,35 @@ mdvs scaffold skill > .agents/skills/mdvs/SKILL.md mdvs scaffold snippet --platform antigravity >> AGENTS.md ``` -Antigravity reads skills from `.agents/skills//SKILL.md` (the cross-harness Agent Skills convention — same path Codex uses) and reads project rules from `AGENTS.md` at the workspace root. +Antigravity reads skills from `.agents/skills//SKILL.md` (the +cross-harness Agent Skills convention — same path Codex uses) and reads project +rules from `AGENTS.md` at the workspace root. ## What you get -- **Skill**: agent learns when to call which mdvs command, how to interpret violations, and the schema-evolution loop. Loaded by Antigravity on session start. -- **Snippet**: always-on project-rules block telling the agent to prefer `mdvs search` over `Grep` for KB lookups. +- **Skill**: agent learns when to call which mdvs command, how to interpret + violations, and the schema-evolution loop. Loaded by Antigravity on session + start. +- **Snippet**: always-on project-rules block telling the agent to prefer + `mdvs search` over `Grep` for KB lookups. ## Hooks -mdvs doesn't ship a verified Antigravity hook config. Antigravity inherits parts of its configuration sources from Gemini CLI, so the hooks system should be compatible with what's documented at the [Gemini CLI hooks reference](https://github.com/google-gemini/gemini-cli/tree/main/docs/hooks). +mdvs doesn't ship a verified Antigravity hook config. Antigravity inherits parts +of its configuration sources from Gemini CLI, so the hooks system should be +compatible with what's documented at the +[Gemini CLI hooks reference](https://github.com/google-gemini/gemini-cli/tree/main/docs/hooks). -As a harness-independent fallback, the [pre-commit hook](../agent-harnesses.md#pre-commit-hook) runs `mdvs check` on every commit. +As a harness-independent fallback, the +[pre-commit hook](../agent-harnesses.md#pre-commit-hook) runs `mdvs check` on +every commit. ## Per-platform notes -- **Skill install path**: `.agents/skills/mdvs/SKILL.md`. Project-scoped path; the agent picks it up when working in the directory. -- **AGENTS.md**: documented as the post-rebrand convention. Legacy Gemini CLI sessions also recognized `GEMINI.md`; both still appear in flux. +- **Skill install path**: `.agents/skills/mdvs/SKILL.md`. Project-scoped path; + the agent picks it up when working in the directory. +- **AGENTS.md**: documented as the post-rebrand convention. Legacy Gemini CLI + sessions also recognized `GEMINI.md`; both still appear in flux. ## Sources diff --git a/book/src/recipes/agent-harnesses/claude-code.md b/book/src/recipes/agent-harnesses/claude-code.md index 0eadb1d..cdb62e4 100644 --- a/book/src/recipes/agent-harnesses/claude-code.md +++ b/book/src/recipes/agent-harnesses/claude-code.md @@ -11,21 +11,36 @@ mdvs scaffold snippet --platform claude-code >> CLAUDE.md mdvs scaffold hook --platform claude-code >> .claude/settings.json # merge into existing hooks ``` -The last command emits a JSON snippet — if `.claude/settings.json` already exists with other settings, **merge by hand** instead of appending blindly: the `hooks.PostToolUse` array should be unioned with anything you already have. mdvs's emitted snippet self-documents the merge target in a `_comment` field at the top. +The last command emits a JSON snippet — if `.claude/settings.json` already +exists with other settings, **merge by hand** instead of appending blindly: the +`hooks.PostToolUse` array should be unioned with anything you already have. +mdvs's emitted snippet self-documents the merge target in a `_comment` field at +the top. ## What you get -- **Skill**: agent learns when to call which mdvs command, how to interpret violations, and the schema-evolution loop. Loaded on session start; activated by description-match or directly via `/mdvs`. -- **Snippet**: always-on `CLAUDE.md` block telling the agent to prefer `mdvs search` over `Grep` for KB lookups. -- **Validate hook**: after every `Edit` / `Write` / `MultiEdit` on a markdown file inside an mdvs vault, `mdvs hook handle` runs `check` and surfaces violations through `additionalContext` (agent-visible) and `systemMessage` (user-visible, capped at 15 lines). Hook always exits 0 — never blocks. -- **Search-nudge hook**: after every `Bash` command that runs `grep` / `rg` / `find` / etc., if the agent's cwd is in an mdvs vault, surfaces a one-line tip pointing at `mdvs search`. +- **Skill**: agent learns when to call which mdvs command, how to interpret + violations, and the schema-evolution loop. Loaded on session start; activated + by description-match or directly via `/mdvs`. +- **Snippet**: always-on `CLAUDE.md` block telling the agent to prefer + `mdvs search` over `Grep` for KB lookups. +- **Validate hook**: after every `Edit` / `Write` / `MultiEdit` on a markdown + file inside an mdvs vault, `mdvs hook handle` runs `check` and surfaces + violations through `additionalContext` (agent-visible) and `systemMessage` + (user-visible, capped at 15 lines). Hook always exits 0 — never blocks. +- **Search-nudge hook**: after every `Bash` command that runs `grep` / `rg` / + `find` / etc., if the agent's cwd is in an mdvs vault, surfaces a one-line tip + pointing at `mdvs search`. ## Per-platform notes -- **Skill path**: `.claude/skills/mdvs/SKILL.md` (Claude Code reads only from `.claude/skills/`, not the cross-harness `.agents/skills/`). +- **Skill path**: `.claude/skills/mdvs/SKILL.md` (Claude Code reads only from + `.claude/skills/`, not the cross-harness `.agents/skills/`). - **Project rules**: `CLAUDE.md` at workspace root. -- **Hook envelope**: Claude Code's `hookSpecificOutput.additionalContext` + `systemMessage` shape. PascalCase event name (`PostToolUse`). -- **mdvs on PATH**: the hook command is `mdvs hook handle --platform claude-code --kind `. +- **Hook envelope**: Claude Code's `hookSpecificOutput.additionalContext` + + `systemMessage` shape. PascalCase event name (`PostToolUse`). +- **mdvs on PATH**: the hook command is + `mdvs hook handle --platform claude-code --kind `. ## Sources diff --git a/book/src/recipes/agent-harnesses/codex.md b/book/src/recipes/agent-harnesses/codex.md index 9158596..f0b82f8 100644 --- a/book/src/recipes/agent-harnesses/codex.md +++ b/book/src/recipes/agent-harnesses/codex.md @@ -10,19 +10,29 @@ mdvs scaffold snippet --platform codex >> AGENTS.md ## What you get -- **Skill**: agent learns when to call which mdvs command, how to interpret violations, and the schema-evolution loop. Loaded from `.agents/skills/mdvs/SKILL.md` (the cross-harness Agent Skills convention). -- **Snippet**: always-on `AGENTS.md` block telling the agent to prefer `mdvs search` over `Grep`. +- **Skill**: agent learns when to call which mdvs command, how to interpret + violations, and the schema-evolution loop. Loaded from + `.agents/skills/mdvs/SKILL.md` (the cross-harness Agent Skills convention). +- **Snippet**: always-on `AGENTS.md` block telling the agent to prefer + `mdvs search` over `Grep`. ## Hooks -mdvs doesn't ship a verified Codex hook config. To wire `mdvs hook handle` into Codex's PostToolUse mechanism, follow the [Codex hooks docs](https://developers.openai.com/codex/hooks). +mdvs doesn't ship a verified Codex hook config. To wire `mdvs hook handle` into +Codex's PostToolUse mechanism, follow the +[Codex hooks docs](https://developers.openai.com/codex/hooks). -As a harness-independent fallback, the [pre-commit hook](../agent-harnesses.md#pre-commit-hook) runs `mdvs check` on every commit. +As a harness-independent fallback, the +[pre-commit hook](../agent-harnesses.md#pre-commit-hook) runs `mdvs check` on +every commit. ## Per-platform notes -- **Skill path**: `.agents/skills/mdvs/SKILL.md` (Codex's canonical path per [their skills docs](https://developers.openai.com/codex/skills/) — same path Cursor and Antigravity also honor). -- **Project rules**: `AGENTS.md` at workspace root. `AGENTS.override.md` takes precedence if present. +- **Skill path**: `.agents/skills/mdvs/SKILL.md` (Codex's canonical path per + [their skills docs](https://developers.openai.com/codex/skills/) — same path + Cursor and Antigravity also honor). +- **Project rules**: `AGENTS.md` at workspace root. `AGENTS.override.md` takes + precedence if present. - **mdvs on PATH**: `mdvs` must be available to any subprocess Codex runs. ## Sources diff --git a/book/src/recipes/agent-harnesses/cursor.md b/book/src/recipes/agent-harnesses/cursor.md index 963878e..ea92029 100644 --- a/book/src/recipes/agent-harnesses/cursor.md +++ b/book/src/recipes/agent-harnesses/cursor.md @@ -10,19 +10,33 @@ mdvs scaffold snippet --platform cursor > .cursor/rules/mdvs.mdc ## What you get -- **Skill**: agent learns when to call which mdvs command, how to interpret violations, and the schema-evolution loop. Cursor reads skills from `.cursor/skills/`, `.agents/skills/`, `.claude/skills/`, and `.codex/skills/` — `mdvs scaffold skill --platform cursor` writes to `.cursor/skills/` as the native path. -- **Snippet**: `.cursor/rules/mdvs.mdc` with `alwaysApply: true`. Cursor includes it in every conversation automatically. +- **Skill**: agent learns when to call which mdvs command, how to interpret + violations, and the schema-evolution loop. Cursor reads skills from + `.cursor/skills/`, `.agents/skills/`, `.claude/skills/`, and `.codex/skills/` + — `mdvs scaffold skill --platform cursor` writes to `.cursor/skills/` as the + native path. +- **Snippet**: `.cursor/rules/mdvs.mdc` with `alwaysApply: true`. Cursor + includes it in every conversation automatically. ## Hooks -mdvs doesn't ship a verified Cursor hook config. To wire `mdvs hook handle` into Cursor's PostToolUse mechanism, follow the [Cursor hooks docs](https://cursor.com/docs/hooks). +mdvs doesn't ship a verified Cursor hook config. To wire `mdvs hook handle` into +Cursor's PostToolUse mechanism, follow the +[Cursor hooks docs](https://cursor.com/docs/hooks). -As a harness-independent fallback, the [pre-commit hook](../agent-harnesses.md#pre-commit-hook) runs `mdvs check` on every commit. +As a harness-independent fallback, the +[pre-commit hook](../agent-harnesses.md#pre-commit-hook) runs `mdvs check` on +every commit. ## Per-platform notes -- **Project rules**: Cursor also honors `AGENTS.md` at workspace root. If you'd rather paste the universal snippet there: `mdvs scaffold snippet >> AGENTS.md` (without `--platform`). -- **mdvs on PATH**: `mdvs` must be available to any subprocess Cursor runs. On macOS, Cursor launched from Spotlight may not see your shell PATH — symlink `mdvs` into `/usr/local/bin/` or install via `cargo install --path crates/mdvs`. +- **Project rules**: Cursor also honors `AGENTS.md` at workspace root. If you'd + rather paste the universal snippet there: `mdvs scaffold snippet >> AGENTS.md` + (without `--platform`). +- **mdvs on PATH**: `mdvs` must be available to any subprocess Cursor runs. On + macOS, Cursor launched from Spotlight may not see your shell PATH — symlink + `mdvs` into `/usr/local/bin/` or install via + `cargo install --path crates/mdvs`. ## Sources diff --git a/book/src/recipes/agent-harnesses/opencode.md b/book/src/recipes/agent-harnesses/opencode.md index 3dbbd78..ac2d4c6 100644 --- a/book/src/recipes/agent-harnesses/opencode.md +++ b/book/src/recipes/agent-harnesses/opencode.md @@ -10,19 +10,30 @@ mdvs scaffold snippet --platform opencode >> AGENTS.md ## What you get -- **Skill**: agent learns when to call which mdvs command, how to interpret violations, and the schema-evolution loop. Loaded by OpenCode on session start. -- **Snippet**: always-on `AGENTS.md` block telling the agent to prefer `mdvs search` over `Grep`. +- **Skill**: agent learns when to call which mdvs command, how to interpret + violations, and the schema-evolution loop. Loaded by OpenCode on session + start. +- **Snippet**: always-on `AGENTS.md` block telling the agent to prefer + `mdvs search` over `Grep`. ## Hooks -OpenCode handles tool events through a TypeScript plugin API rather than shell-command hooks. At the moment, mdvs doesn't ship a verified plugin or hook config for OpenCode, to wire `mdvs hook handle` into OpenCode's plugin events, follow the [OpenCode docs](https://opencode.ai/docs/). +OpenCode handles tool events through a TypeScript plugin API rather than +shell-command hooks. At the moment, mdvs doesn't ship a verified plugin or hook +config for OpenCode, to wire `mdvs hook handle` into OpenCode's plugin events, +follow the [OpenCode docs](https://opencode.ai/docs/). -As a harness-independent fallback, the [pre-commit hook](../agent-harnesses.md#pre-commit-hook) runs `mdvs check` on every commit. +As a harness-independent fallback, the +[pre-commit hook](../agent-harnesses.md#pre-commit-hook) runs `mdvs check` on +every commit. ## Per-platform notes -- **Skill path**: `.opencode/skills/mdvs/SKILL.md` (native). OpenCode also reads `.claude/skills/` and `.agents/skills/` — you can symlink across if you want a single source of truth shared with another harness. -- **Project rules**: `AGENTS.md` at workspace root. OpenCode also reads `CLAUDE.md` as a Claude Code-compat fallback. +- **Skill path**: `.opencode/skills/mdvs/SKILL.md` (native). OpenCode also reads + `.claude/skills/` and `.agents/skills/` — you can symlink across if you want a + single source of truth shared with another harness. +- **Project rules**: `AGENTS.md` at workspace root. OpenCode also reads + `CLAUDE.md` as a Claude Code-compat fallback. ## Sources diff --git a/book/src/recipes/ci.md b/book/src/recipes/ci.md index 055ee6c..591d48d 100644 --- a/book/src/recipes/ci.md +++ b/book/src/recipes/ci.md @@ -1,6 +1,9 @@ # CI -`mdvs check` exits with code 1 when any file violates the schema, so it slots straight into a CI pipeline as a frontmatter linter. This page covers the GitHub Actions case, but the same shape works on GitLab CI, CircleCI, or any runner that can install a binary and run a command. +`mdvs check` exits with code 1 when any file violates the schema, so it slots +straight into a CI pipeline as a frontmatter linter. This page covers the GitHub +Actions case, but the same shape works on GitLab CI, CircleCI, or any runner +that can install a binary and run a command. ## Minimal GitHub Actions workflow @@ -31,46 +34,84 @@ jobs: run: mdvs check --no-update ``` -Replace `vX.Y.Z` with a real release tag (see the [releases page](https://github.com/edochi/mdvs/releases)). This adds a check that runs on every PR and every push to `main`. If a contributor introduces a file with a wrong type, missing required field, disallowed field, or unrepresentable frontmatter, the job fails and the PR is blocked until it's fixed. +Replace `vX.Y.Z` with a real release tag (see the +[releases page](https://github.com/edochi/mdvs/releases)). This adds a check +that runs on every PR and every push to `main`. If a contributor introduces a +file with a wrong type, missing required field, disallowed field, or +unrepresentable frontmatter, the job fails and the PR is blocked until it's +fixed. ## Pin the mdvs version -The installer URL above pulls a **specific release tag**. GitHub also exposes a `releases/latest/download/...` URL that always redirects to the newest release — convenient for casual use, and that's what the [README install snippet](https://github.com/edochi/mdvs#install) uses — but in CI you want **reproducibility**. Pinning a specific tag means a green check today still passes (or still fails the same way) tomorrow, regardless of what mdvs ships in the meantime. +The installer URL above pulls a **specific release tag**. GitHub also exposes a +`releases/latest/download/...` URL that always redirects to the newest release — +convenient for casual use, and that's what the +[README install snippet](https://github.com/edochi/mdvs#install) uses — but in +CI you want **reproducibility**. Pinning a specific tag means a green check +today still passes (or still fails the same way) tomorrow, regardless of what +mdvs ships in the meantime. -Bump the pinned version when you're ready to adopt new validation behavior. The mdvs [release notes](https://github.com/edochi/mdvs/releases) call out anything that affects validation output. +Bump the pinned version when you're ready to adopt new validation behavior. The +mdvs [release notes](https://github.com/edochi/mdvs/releases) call out anything +that affects validation output. ## `--no-update` for deterministic CI -The `--no-update` flag (or `[check].auto_update = false` in `mdvs.toml`) tells `check` to validate against the committed schema instead of re-running inference first. This matters in CI: +The `--no-update` flag (or `[check].auto_update = false` in `mdvs.toml`) tells +`check` to validate against the committed schema instead of re-running inference +first. This matters in CI: -- **With auto-update on:** `check` re-infers the schema before validating and **rewrites `mdvs.toml` on disk**, absorbing any new frontmatter field. On a runner that rewrite is thrown away with the checkout, but the run validated against a schema that differs from the committed one, and the new field is never mentioned. -- **With `--no-update`:** `mdvs.toml` is left untouched and the run validates against exactly what's committed. A new field is reported under a **New fields** heading in the output. +- **With auto-update on:** `check` re-infers the schema before validating and + **rewrites `mdvs.toml` on disk**, absorbing any new frontmatter field. On a + runner that rewrite is thrown away with the checkout, but the run validated + against a schema that differs from the committed one, and the new field is + never mentioned. +- **With `--no-update`:** `mdvs.toml` is left untouched and the run validates + against exactly what's committed. A new field is reported under a **New + fields** heading in the output. -Re-inference only ever *adds* fields — it does not widen an existing field's type or relax its constraints. A value that breaks a declared `categories` list, or a field with the wrong type, fails in both modes. +Re-inference only ever _adds_ fields — it does not widen an existing field's +type or relax its constraints. A value that breaks a declared `categories` list, +or a field with the wrong type, fails in both modes. -In practice: in CI, **always** use `--no-update`. You validate against the committed schema, nothing is rewritten mid-run, and additions are visible in the log. +In practice: in CI, **always** use `--no-update`. You validate against the +committed schema, nothing is rewritten mid-run, and additions are visible in the +log. ### `check` does not fail on undeclared fields -Worth being explicit, because it is easy to assume otherwise: a frontmatter field that appears in **no** `[[fields.field]]` entry is **not** a violation, in either mode. `--no-update` surfaces it as informational output and the command still **exits 0**. +Worth being explicit, because it is easy to assume otherwise: a frontmatter +field that appears in **no** `[[fields.field]]` entry is **not** a violation, in +either mode. `--no-update` surfaces it as informational output and the command +still **exits 0**. ``` Checked 2 files — no violations, 1 new field(s) ``` -Validation iterates the fields declared in `mdvs.toml`, so a key it has never seen is reported, not rejected. `Disallowed` means something narrower: a *declared* field appearing at a path outside its `allowed` globs. +Validation iterates the fields declared in `mdvs.toml`, so a key it has never +seen is reported, not rejected. `Disallowed` means something narrower: a +_declared_ field appearing at a path outside its `allowed` globs. -If you want new fields to break the build today, gate on the JSON output — it carries a `new_fields` array: +If you want new fields to break the build today, gate on the JSON output — it +carries a `new_fields` array: ```bash mdvs check --no-update --output json | jq -e '.new_fields | length == 0' ``` -`jq -e` exits non-zero when the expression is false, so the step fails as soon as an undeclared field shows up. Run it *in addition to* `mdvs check --no-update`, which still owns the real violations. A schema-level "freeze this directory" option does not exist yet. +`jq -e` exits non-zero when the expression is false, so the step fails as soon +as an undeclared field shows up. Run it _in addition to_ +`mdvs check --no-update`, which still owns the real violations. A schema-level +"freeze this directory" option does not exist yet. ## Caching the install -The installer step downloads a small binary (~6 MB on Linux) and finishes in well under a second. There's usually no point caching it. If you want to avoid the network call entirely on every run, use `actions/cache` keyed on the mdvs version string, or commit a vendored binary into the repo and skip the install step. +The installer step downloads a small binary (~6 MB on Linux) and finishes in +well under a second. There's usually no point caching it. If you want to avoid +the network call entirely on every run, use `actions/cache` keyed on the mdvs +version string, or commit a vendored binary into the repo and skip the install +step. ## What `check` does (and doesn't) @@ -81,16 +122,30 @@ The installer step downloads a small binary (~6 MB on Linux) and finishes in wel - ✓ Disallowed fields (a declared field appearing outside its `allowed` paths) - ✓ Null violations - ✓ Category, length, range, and regex constraint violations -- ✓ Frontmatter that can't be parsed at all (broken YAML, broken TOML, broken JSON) +- ✓ Frontmatter that can't be parsed at all (broken YAML, broken TOML, broken + JSON) -It does **not** flag a field that appears in no `[[fields.field]]` entry — see [above](#check-does-not-fail-on-undeclared-fields). It also does **not** check spelling, link validity, markdown style, or anything in the body content. Pair it with a markdown linter (markdownlint, vale) for those concerns. They run independently and have no conflict — `mdvs check` and a body-content linter cover orthogonal parts of the file. +It does **not** flag a field that appears in no `[[fields.field]]` entry — see +[above](#check-does-not-fail-on-undeclared-fields). It also does **not** check +spelling, link validity, markdown style, or anything in the body content. Pair +it with a markdown linter (markdownlint, vale) for those concerns. They run +independently and have no conflict — `mdvs check` and a body-content linter +cover orthogonal parts of the file. ## Other CI systems The shape translates directly: -- **GitLab CI:** the same two-step install-then-run pattern in `.gitlab-ci.yml`. Use the install script under `before_script:` and run `mdvs check --no-update` in the job. -- **CircleCI:** an `orb` or a custom step that installs the binary and invokes the check. -- **Pre-commit hook:** `mdvs check --no-update` as a hook entry in `.pre-commit-config.yaml` runs the check locally on every commit, catching issues before they reach CI. See the dedicated [pre-commit recipe](./pre-commit.md) for both the framework and plain-git-hook setups. - -The contract is always the same: install `mdvs`, run `mdvs check --no-update`, fail on non-zero exit. +- **GitLab CI:** the same two-step install-then-run pattern in `.gitlab-ci.yml`. + Use the install script under `before_script:` and run `mdvs check --no-update` + in the job. +- **CircleCI:** an `orb` or a custom step that installs the binary and invokes + the check. +- **Pre-commit hook:** `mdvs check --no-update` as a hook entry in + `.pre-commit-config.yaml` runs the check locally on every commit, catching + issues before they reach CI. See the dedicated + [pre-commit recipe](./pre-commit.md) for both the framework and plain-git-hook + setups. + +The contract is always the same: install `mdvs`, run `mdvs check --no-update`, +fail on non-zero exit. diff --git a/book/src/recipes/hugo.md b/book/src/recipes/hugo.md index ac559a2..e95a9f0 100644 --- a/book/src/recipes/hugo.md +++ b/book/src/recipes/hugo.md @@ -1,6 +1,9 @@ # Hugo -mdvs works directly on a [Hugo](https://gohugo.io/) site's `content/` tree. Hugo accepts YAML (`---`), TOML (`+++`), and JSON (`{...}`) frontmatter; mdvs accepts the same three formats and auto-detects per file, so it doesn't matter which convention your site uses — or whether you've drifted across formats over time. +mdvs works directly on a [Hugo](https://gohugo.io/) site's `content/` tree. Hugo +accepts YAML (`---`), TOML (`+++`), and JSON (`{...}`) frontmatter; mdvs accepts +the same three formats and auto-detects per file, so it doesn't matter which +convention your site uses — or whether you've drifted across formats over time. ## Setup @@ -10,14 +13,19 @@ Point mdvs at the `content/` directory: mdvs init path/to/site/content ``` -This scans every markdown file, infers a typed schema from the frontmatter (across all three formats), and writes `mdvs.toml` alongside. If auto-build is enabled (the default), it also downloads the embedding model and builds the search index under `.mdvs/`. +This scans every markdown file, infers a typed schema from the frontmatter +(across all three formats), and writes `mdvs.toml` alongside. If auto-build is +enabled (the default), it also downloads the embedding model and builds the +search index under `.mdvs/`. Two artifacts are created next to `content/`: - **`mdvs.toml`** — commit to version control - **`.mdvs/`** — add to `.gitignore` (search index, regenerable) -Some Hugo sites prefer to keep the schema and index alongside the site root rather than inside `content/`. In that case, run `mdvs init .` from the site root and use a glob: +Some Hugo sites prefer to keep the schema and index alongside the site root +rather than inside `content/`. In that case, run `mdvs init .` from the site +root and use a glob: ```toml [scan] @@ -26,7 +34,12 @@ glob = "content/**" ## Mixed-format vaults -Hugo's docs show all three frontmatter formats interchangeably, and real-world sites often end up with a mix — an older `---` post sitting next to a newer `+++` post and an occasional `{...}` block emitted by a content tool. mdvs handles this transparently. A single `mdvs.toml` is inferred across all three formats; the same `title`, `tags`, `draft` fields collapse into one schema regardless of where they were written. +Hugo's docs show all three frontmatter formats interchangeably, and real-world +sites often end up with a mix — an older `---` post sitting next to a newer +`+++` post and an occasional `{...}` block emitted by a content tool. mdvs +handles this transparently. A single `mdvs.toml` is inferred across all three +formats; the same `title`, `tags`, `draft` fields collapse into one schema +regardless of where they were written. You can verify this with `mdvs check` after init: @@ -37,18 +50,23 @@ Checked 142 files — no violations ## Forcing a single format -If your site is opinionated about TOML (Hugo's default for `hugo new`), tell mdvs: +If your site is opinionated about TOML (Hugo's default for `hugo new`), tell +mdvs: ```toml [scan] frontmatter_format = "toml" ``` -Now any file that uses `---` (YAML) or `{` (JSON) raises a `FrontmatterUnrepresentable` error during `check`, naming both the configured and detected delimiters. Useful when you want your CI to fail loudly if someone drops in a YAML post by accident. +Now any file that uses `---` (YAML) or `{` (JSON) raises a +`FrontmatterUnrepresentable` error during `check`, naming both the configured +and detected delimiters. Useful when you want your CI to fail loudly if someone +drops in a YAML post by accident. ## Native TOML dates -Hugo's TOML frontmatter often uses native `Date` / `DateTime` literals — unquoted, e.g.: +Hugo's TOML frontmatter often uses native `Date` / `DateTime` literals — +unquoted, e.g.: ```toml +++ @@ -58,7 +76,9 @@ publishedAt = 2024-09-01T09:00:00Z +++ ``` -mdvs recognizes both as typed fields: `date` becomes `FieldType::Date`, `publishedAt` becomes `FieldType::DateTime`. No special configuration. You can then filter on them in search: +mdvs recognizes both as typed fields: `date` becomes `FieldType::Date`, +`publishedAt` becomes `FieldType::DateTime`. No special configuration. You can +then filter on them in search: ```bash mdvs search "release notes" --where "publishedAt > '2024-01-01T00:00:00Z'" @@ -87,7 +107,9 @@ mdvs search "authentication" \ --where "author = 'alice' AND date >= '2024-01-01' AND date < '2024-04-01'" ``` -The `--where` clause is SQL against your frontmatter — anything you can express as a column reference works. See the [Search Guide](../search-guide.md) for details. +The `--where` clause is SQL against your frontmatter — anything you can express +as a column reference works. See the [Search Guide](../search-guide.md) for +details. ## Validating across an editorial workflow @@ -101,6 +123,9 @@ Add `mdvs check` to your Hugo build pipeline so frontmatter drift fails CI: run: hugo --minify ``` -`mdvs check` returns exit code 1 if any file violates the schema (missing required field, wrong type, etc.), which is enough to break the build. The exact same `mdvs.toml` validates YAML, TOML, and JSON files uniformly — no per-format duplicate rules. +`mdvs check` returns exit code 1 if any file violates the schema (missing +required field, wrong type, etc.), which is enough to break the build. The exact +same `mdvs.toml` validates YAML, TOML, and JSON files uniformly — no per-format +duplicate rules. See the [CI recipe](./ci.md) for a more general-purpose CI workflow. diff --git a/book/src/recipes/obsidian.md b/book/src/recipes/obsidian.md index f738134..c85f3d1 100644 --- a/book/src/recipes/obsidian.md +++ b/book/src/recipes/obsidian.md @@ -1,6 +1,10 @@ # Obsidian -mdvs works well with [Obsidian](https://obsidian.md/) vaults — it validates your YAML frontmatter for consistency and provides semantic search across all your notes. Everything runs locally, no external services needed. (Obsidian emits YAML; mdvs also handles TOML and JSON if you've imported notes from other tools — see the [Hugo recipe](./hugo.md) for the mixed-format case.) +mdvs works well with [Obsidian](https://obsidian.md/) vaults — it validates your +YAML frontmatter for consistency and provides semantic search across all your +notes. Everything runs locally, no external services needed. (Obsidian emits +YAML; mdvs also handles TOML and JSON if you've imported notes from other tools +— see the [Hugo recipe](./hugo.md) for the mixed-format case.) ## Setup @@ -10,7 +14,9 @@ Point mdvs at your vault: mdvs init path/to/vault ``` -This scans all markdown files, infers a typed schema from your frontmatter, and writes `mdvs.toml`. If auto-build is enabled (the default), it also downloads the embedding model and builds the search index. +This scans all markdown files, infers a typed schema from your frontmatter, and +writes `mdvs.toml`. If auto-build is enabled (the default), it also downloads +the embedding model and builds the search index. Two artifacts are created: @@ -19,11 +25,14 @@ Two artifacts are created: ### .gitignore -mdvs respects `.gitignore` by default. If your vault has `.obsidian/` in `.gitignore` (many do), those files are automatically excluded from scanning. No extra configuration needed. +mdvs respects `.gitignore` by default. If your vault has `.obsidian/` in +`.gitignore` (many do), those files are automatically excluded from scanning. No +extra configuration needed. ### .mdvsignore -For additional exclusions, create a `.mdvsignore` file at the vault root. It uses the same syntax as `.gitignore`: +For additional exclusions, create a `.mdvsignore` file at the vault root. It +uses the same syntax as `.gitignore`: ``` # AI working directories @@ -38,7 +47,8 @@ attachments/ assets/ ``` -Any directory that doesn't contain markdown with frontmatter is a good candidate for exclusion — it speeds up scanning and avoids noise in the schema. +Any directory that doesn't contain markdown with frontmatter is a good candidate +for exclusion — it speeds up scanning and avoids noise in the schema. ## Common frontmatter patterns @@ -56,21 +66,27 @@ draft: false mdvs infers types automatically: -| Field | Inferred type | Notes | -|---|---|---| -| `title` | String | | -| `tags` | Array(String) | Array of strings | -| `status` | String | | -| `date` | Date | RFC 3339 `YYYY-MM-DD` strings auto-promote to `Date`; mixed shapes fall back to `String` | -| `draft` | Boolean | | +| Field | Inferred type | Notes | +| -------- | ------------- | ---------------------------------------------------------------------------------------- | +| `title` | String | | +| `tags` | Array(String) | Array of strings | +| `status` | String | | +| `date` | Date | RFC 3339 `YYYY-MM-DD` strings auto-promote to `Date`; mixed shapes fall back to `String` | +| `draft` | Boolean | | ### Inconsistent types -If the same field has different types across notes (e.g., `priority` is an integer in some files and a string like `"high"` in others), mdvs widens to the broadest compatible type — usually String. See [Types & Widening](../concepts/types.md) for the full rules. +If the same field has different types across notes (e.g., `priority` is an +integer in some files and a string like `"high"` in others), mdvs widens to the +broadest compatible type — usually String. See +[Types & Widening](../concepts/types.md) for the full rules. ### Dataview fields -If you use the [Dataview](https://blacksmithgu.github.io/obsidian-dataview/) plugin, its inline fields (e.g., `key:: value`) are **not** picked up by mdvs — only YAML frontmatter between `---` fences is scanned. Dataview fields that appear in the YAML block are handled normally. +If you use the [Dataview](https://blacksmithgu.github.io/obsidian-dataview/) +plugin, its inline fields (e.g., `key:: value`) are **not** picked up by mdvs — +only YAML frontmatter between `---` fences is scanned. Dataview fields that +appear in the YAML block are handled normally. ## Validation @@ -83,7 +99,8 @@ mdvs check path/to/vault This catches: - **Wrong types** — a Boolean field with a string value -- **Missing required fields** — a field that should be present in certain directories +- **Missing required fields** — a field that should be present in certain + directories - **Disallowed fields** — a field appearing where it shouldn't - **Null violations** — null where it's not allowed @@ -91,7 +108,8 @@ See [Validation](../concepts/validation.md) for the full rules. ### Tightening constraints -The inferred schema is permissive by default. To enforce stricter rules, edit `mdvs.toml` directly. For example, to require `tags` in all daily notes: +The inferred schema is permissive by default. To enforce stricter rules, edit +`mdvs.toml` directly. For example, to require `tags` in all daily notes: ```toml [[fields.field]] @@ -110,7 +128,9 @@ When you introduce new frontmatter fields, run `update` to incorporate them: mdvs update path/to/vault ``` -This discovers new fields and adds them to `mdvs.toml` without touching existing field definitions. Use the `reinfer` subcommand to re-infer specific fields if you've reorganized your vault. +This discovers new fields and adds them to `mdvs.toml` without touching existing +field definitions. Use the `reinfer` subcommand to re-infer specific fields if +you've reorganized your vault. ## Search @@ -138,13 +158,23 @@ See the [Search Guide](../search-guide.md) for the full `--where` reference. ## Tips -- **Incremental builds** — only notes whose body changed since the last build are re-embedded. Frontmatter-only changes (updating tags, status) don't trigger re-embedding. Run `mdvs build` freely — on an unchanged vault the index write itself is skipped, so it's effectively a no-op. +- **Incremental builds** — only notes whose body changed since the last build + are re-embedded. Frontmatter-only changes (updating tags, status) don't + trigger re-embedding. Run `mdvs build` freely — on an unchanged vault the + index write itself is skipped, so it's effectively a no-op. -- **Alongside Obsidian search** — mdvs search is semantic (finds conceptually related notes), while Obsidian's built-in search is keyword-based. They complement each other. +- **Alongside Obsidian search** — mdvs search is semantic (finds conceptually + related notes), while Obsidian's built-in search is keyword-based. They + complement each other. -- **Large vaults** — mdvs has been tested on vaults of over 1,500 files; full build from scratch finishes in single-digit seconds, and incremental builds touching one or two files complete in tens of milliseconds. See [docs/benchmarks/](https://github.com/edochi/mdvs/tree/main/docs/benchmarks) for measured numbers. +- **Large vaults** — mdvs has been tested on vaults of over 1,500 files; full + build from scratch finishes in single-digit seconds, and incremental builds + touching one or two files complete in tens of milliseconds. See + [docs/benchmarks/](https://github.com/edochi/mdvs/tree/main/docs/benchmarks) + for measured numbers. -- **Ignore noisy fields** — if some frontmatter fields are auto-generated and you don't want to validate them, add them to the `ignore` list in `mdvs.toml`: +- **Ignore noisy fields** — if some frontmatter fields are auto-generated and + you don't want to validate them, add them to the `ignore` list in `mdvs.toml`: ```toml [fields] ignore = ["cssclass", "kanban-plugin"] diff --git a/book/src/recipes/pre-commit.md b/book/src/recipes/pre-commit.md index 4e2f2c8..d99f28f 100644 --- a/book/src/recipes/pre-commit.md +++ b/book/src/recipes/pre-commit.md @@ -1,20 +1,38 @@ # Pre-commit hook -`mdvs check` exits non-zero when any file violates the schema, so it works as a git pre-commit hook: run it before each commit and frontmatter mistakes never reach the repo. This is the local counterpart to the [CI recipe](./ci.md) — same command, same contract, one step earlier in the loop. Catching a violation at commit time is faster than waiting for a CI job to fail on the pushed branch. +`mdvs check` exits non-zero when any file violates the schema, so it works as a +git pre-commit hook: run it before each commit and frontmatter mistakes never +reach the repo. This is the local counterpart to the [CI recipe](./ci.md) — same +command, same contract, one step earlier in the loop. Catching a violation at +commit time is faster than waiting for a CI job to fail on the pushed branch. -There are two ways to wire it up: the [pre-commit framework](https://pre-commit.com) (if you already use it) or a plain git hook script (zero dependencies). Both run the same command. +There are two ways to wire it up: the +[pre-commit framework](https://pre-commit.com) (if you already use it) or a +plain git hook script (zero dependencies). Both run the same command. ## Use `--no-update` -Every setup below runs `mdvs check --no-update`. The flag validates against the committed `mdvs.toml` instead of re-running inference first. +Every setup below runs `mdvs check --no-update`. The flag validates against the +committed `mdvs.toml` instead of re-running inference first. -This matters more in a hook than in CI. Without it, `check` **rewrites `mdvs.toml` on disk** as it runs — so a commit that adds a new frontmatter field silently edits your schema file mid-commit, leaving a modified `mdvs.toml` in your working tree that isn't part of what you staged. With it, the file is left alone and the new field is reported instead. +This matters more in a hook than in CI. Without it, `check` **rewrites +`mdvs.toml` on disk** as it runs — so a commit that adds a new frontmatter field +silently edits your schema file mid-commit, leaving a modified `mdvs.toml` in +your working tree that isn't part of what you staged. With it, the file is left +alone and the new field is reported instead. -Note what `--no-update` does *not* do: an undeclared field is **not** a violation, and the hook still exits 0, so it won't block the commit. See [`check` does not fail on undeclared fields](./ci.md#check-does-not-fail-on-undeclared-fields) for what does and doesn't gate. The [CI recipe covers the flag in full](./ci.md#--no-update-for-deterministic-ci); the short version is: **always use `--no-update` in a hook.** +Note what `--no-update` does _not_ do: an undeclared field is **not** a +violation, and the hook still exits 0, so it won't block the commit. See +[`check` does not fail on undeclared fields](./ci.md#check-does-not-fail-on-undeclared-fields) +for what does and doesn't gate. The +[CI recipe covers the flag in full](./ci.md#--no-update-for-deterministic-ci); +the short version is: **always use `--no-update` in a hook.** ## pre-commit framework -To install the `pre-commit` tool itself, see [its install docs](https://pre-commit.com/#install), or use [`uv`](https://docs.astral.sh/uv/): +To install the `pre-commit` tool itself, see +[its install docs](https://pre-commit.com/#install), or use +[`uv`](https://docs.astral.sh/uv/): ```bash uv tool install pre-commit @@ -30,7 +48,10 @@ repos: - id: mdvs-check ``` -Replace `vX.Y.Z` with a real [release tag](https://github.com/edochi/mdvs/releases). This references the [`.pre-commit-hooks.yaml`](https://github.com/edochi/mdvs/blob/main/.pre-commit-hooks.yaml) shipped in the mdvs repo, which declares the hook as: +Replace `vX.Y.Z` with a real +[release tag](https://github.com/edochi/mdvs/releases). This references the +[`.pre-commit-hooks.yaml`](https://github.com/edochi/mdvs/blob/main/.pre-commit-hooks.yaml) +shipped in the mdvs repo, which declares the hook as: ```yaml - id: mdvs-check @@ -41,29 +62,51 @@ Replace `vX.Y.Z` with a real [release tag](https://github.com/edochi/mdvs/releas pass_filenames: false ``` -`language: system` means the hook expects `mdvs` to already be on `PATH` — pre-commit will not build it from source. Install mdvs once (see the [README install snippet](https://github.com/edochi/mdvs#install)), then activate the hook: +`language: system` means the hook expects `mdvs` to already be on `PATH` — +pre-commit will not build it from source. Install mdvs once (see the +[README install snippet](https://github.com/edochi/mdvs#install)), then activate +the hook: ```bash pre-commit install ``` -The next `git commit` runs `mdvs check`; if there are violations the commit aborts and the violation report is printed. To run the check manually without committing: +The next `git commit` runs `mdvs check`; if there are violations the commit +aborts and the violation report is printed. To run the check manually without +committing: ```bash pre-commit run --all-files ``` -`pass_filenames: false` is deliberate: `mdvs check` validates the whole vault, not a file list, because rules like "required field per directory" need the full tree. `types: [markdown]` scopes the trigger so the hook only fires when a commit touches markdown. +`pass_filenames: false` is deliberate: `mdvs check` validates the whole vault, +not a file list, because rules like "required field per directory" need the full +tree. `types: [markdown]` scopes the trigger so the hook only fires when a +commit touches markdown. ### Notes -- **Works with any install method.** `language: system` just runs the `mdvs` already on your PATH — it doesn't matter whether you installed via `cargo install mdvs`, the release shell installer, Homebrew, or a manually-placed binary. The only requirement is that `mdvs` is invocable from git's environment. -- **PATH gotcha for GUI git clients.** git hooks fire under git's environment, which isn't always the same as your interactive shell's PATH. If `mdvs` lives in `~/.cargo/bin/` and you commit from a GUI client that doesn't inherit your shell PATH, the hook fails with `mdvs: command not found`. Either commit from the terminal, or use an absolute path (`entry: /Users/you/.cargo/bin/mdvs check --no-update`). The same applies to the plain git hook below. -- **Version-pinned alternative.** To have `pre-commit` fetch `mdvs` into its own isolated environment (slower per-repo install, but reproducible across machines and CI), define the hook inline with `repo: local`, `language: rust`, and `additional_dependencies: ["mdvs"]` instead of referencing this repo. +- **Works with any install method.** `language: system` just runs the `mdvs` + already on your PATH — it doesn't matter whether you installed via + `cargo install mdvs`, the release shell installer, Homebrew, or a + manually-placed binary. The only requirement is that `mdvs` is invocable from + git's environment. +- **PATH gotcha for GUI git clients.** git hooks fire under git's environment, + which isn't always the same as your interactive shell's PATH. If `mdvs` lives + in `~/.cargo/bin/` and you commit from a GUI client that doesn't inherit your + shell PATH, the hook fails with `mdvs: command not found`. Either commit from + the terminal, or use an absolute path + (`entry: /Users/you/.cargo/bin/mdvs check --no-update`). The same applies to + the plain git hook below. +- **Version-pinned alternative.** To have `pre-commit` fetch `mdvs` into its own + isolated environment (slower per-repo install, but reproducible across + machines and CI), define the hook inline with `repo: local`, `language: rust`, + and `additional_dependencies: ["mdvs"]` instead of referencing this repo. ## Plain git hook -No framework, no dependencies — just a script git runs before each commit. Write it to `.git/hooks/pre-commit`: +No framework, no dependencies — just a script git runs before each commit. Write +it to `.git/hooks/pre-commit`: ```bash #!/bin/sh @@ -79,9 +122,14 @@ EOF chmod +x .git/hooks/pre-commit ``` -This is the whole thing. `mdvs check --no-update` exits 1 on a schema violation and 2 on an internal error; either non-zero status aborts the commit. On a clean vault it exits 0 and the commit proceeds. +This is the whole thing. `mdvs check --no-update` exits 1 on a schema violation +and 2 on an internal error; either non-zero status aborts the commit. On a clean +vault it exits 0 and the commit proceeds. -The one caveat: `.git/hooks/` is not version-controlled, so this script lives only in your local clone — each contributor sets it up themselves. If you want the hook shared across a team automatically, use the pre-commit framework above (its config *is* committed) or point `core.hooksPath` at a tracked directory: +The one caveat: `.git/hooks/` is not version-controlled, so this script lives +only in your local clone — each contributor sets it up themselves. If you want +the hook shared across a team automatically, use the pre-commit framework above +(its config _is_ committed) or point `core.hooksPath` at a tracked directory: ```bash git config core.hooksPath .githooks # commit your hook script under .githooks/ @@ -89,8 +137,19 @@ git config core.hooksPath .githooks # commit your hook script under .githooks/ ## Scope -The hook runs `mdvs check` over the whole vault, not just the files staged for the commit. That's the same trade-off the [CI recipe](./ci.md) makes: whole-vault is simpler and catches cross-file violations (a missing required field, a duplicate that only conflicts in aggregate), at the cost of also re-validating files the commit didn't touch. - -In practice that cost is negligible — validation is a frontmatter pass, not an embedding pass, and runs in milliseconds even on vaults of a few thousand files. Validating only the staged files would need a per-file validation mode, which mdvs does not expose: `check` takes a vault path, not a file list. If you have a vault where the whole-vault pass is actually too slow, open an issue. - -Like in CI, `mdvs check` covers frontmatter only — types, required fields, disallowed fields, nulls, constraints, and unparseable frontmatter. It does not check body content, spelling, or links. Pair it with a markdown linter for those; they run independently. +The hook runs `mdvs check` over the whole vault, not just the files staged for +the commit. That's the same trade-off the [CI recipe](./ci.md) makes: +whole-vault is simpler and catches cross-file violations (a missing required +field, a duplicate that only conflicts in aggregate), at the cost of also +re-validating files the commit didn't touch. + +In practice that cost is negligible — validation is a frontmatter pass, not an +embedding pass, and runs in milliseconds even on vaults of a few thousand files. +Validating only the staged files would need a per-file validation mode, which +mdvs does not expose: `check` takes a vault path, not a file list. If you have a +vault where the whole-vault pass is actually too slow, open an issue. + +Like in CI, `mdvs check` covers frontmatter only — types, required fields, +disallowed fields, nulls, constraints, and unparseable frontmatter. It does not +check body content, spelling, or links. Pair it with a markdown linter for +those; they run independently. diff --git a/book/src/search-guide.md b/book/src/search-guide.md index a580169..1d0476d 100644 --- a/book/src/search-guide.md +++ b/book/src/search-guide.md @@ -1,10 +1,21 @@ # Search Guide -The `--where` flag on [search](./commands/search.md) lets you filter results using SQL syntax. The filter is combined with similarity ranking in a single query — files that don't match are excluded before results are returned. `--where` operates on **any column** in the Lance index: frontmatter fields (auto-discovered from `mdvs.toml`) and the always-present `filepath` column (see [Filtering by file path](#filtering-by-file-path)). - -Under the hood, mdvs hands the clause to [LanceDB](https://lancedb.com/)'s SQL filter, which is built on top of DataFusion — so any expression valid in DataFusion's SQL dialect works in `--where`. - -> **Limitation.** `--where` clauses that reference an `Array(Float)` field (e.g. `measurement_values`) are rejected up front, because the underlying search engine can't safely decode them and crashes on read. mdvs catches this before the query runs and returns a clear error. Filter on a scalar field, or store the data as a parallel array of strings, instead. +The `--where` flag on [search](./commands/search.md) lets you filter results +using SQL syntax. The filter is combined with similarity ranking in a single +query — files that don't match are excluded before results are returned. +`--where` operates on **any column** in the Lance index: frontmatter fields +(auto-discovered from `mdvs.toml`) and the always-present `filepath` column (see +[Filtering by file path](#filtering-by-file-path)). + +Under the hood, mdvs hands the clause to [LanceDB](https://lancedb.com/)'s SQL +filter, which is built on top of DataFusion — so any expression valid in +DataFusion's SQL dialect works in `--where`. + +> **Limitation.** `--where` clauses that reference an `Array(Float)` field (e.g. +> `measurement_values`) are rejected up front, because the underlying search +> engine can't safely decode them and crashes on read. mdvs catches this before +> the query runs and returns a clear error. Filter on a scalar field, or store +> the data as a parallel array of strings, instead. ## Scalar fields @@ -73,7 +84,10 @@ mdvs search "notes" --where "NOT status = 'archived'" ## Date and DateTime -Fields typed as `Date` (Arrow `Date32`) and `DateTime` (Arrow `Timestamp(Millisecond, UTC)`) support native date arithmetic, comparisons, and the usual SQL date functions. Auto-inferred from RFC 3339 strings — see [Date and DateTime](./concepts/types.md#date-and-datetime) for the type itself. +Fields typed as `Date` (Arrow `Date32`) and `DateTime` (Arrow +`Timestamp(Millisecond, UTC)`) support native date arithmetic, comparisons, and +the usual SQL date functions. Auto-inferred from RFC 3339 strings — see +[Date and DateTime](./concepts/types.md#date-and-datetime) for the type itself. ### Direct comparison @@ -83,7 +97,9 @@ mdvs search "meeting" --where "date < '2032-01-01'" mdvs search "calibration" --where "synced_at >= '2024-04-01T00:00:00Z'" ``` -DateTime offsets are normalized to UTC at storage time, so `2024-04-02T16:14:30+02:00` (in a YAML file) and `2024-04-02T14:14:30Z` (in a `--where` clause) compare as the same absolute moment. +DateTime offsets are normalized to UTC at storage time, so +`2024-04-02T16:14:30+02:00` (in a YAML file) and `2024-04-02T14:14:30Z` (in a +`--where` clause) compare as the same absolute moment. ### Range filters (`BETWEEN`) @@ -94,7 +110,8 @@ mdvs search "report" --where "joined BETWEEN '2023-01-01' AND '2024-12-31'" ### Date functions (`EXTRACT`, `date_part`) -Both extract numeric components from `Date` and `DateTime`. Two equivalent syntaxes: +Both extract numeric components from `Date` and `DateTime`. Two equivalent +syntaxes: ```bash mdvs search "meeting" --where "EXTRACT(YEAR FROM date) = 2031" @@ -116,7 +133,9 @@ mdvs search "experiment" \ --where "synced_at < CAST('2024-04-15T00:00:00Z' AS TIMESTAMP) - INTERVAL '7 days'" ``` -`CAST('...' AS DATE)` and `CAST('...' AS TIMESTAMP)` are usually needed for string literals on the right side of the arithmetic — the SQL type inference doesn't always pick the date/timestamp type automatically. +`CAST('...' AS DATE)` and `CAST('...' AS TIMESTAMP)` are usually needed for +string literals on the right side of the arithmetic — the SQL type inference +doesn't always pick the date/timestamp type automatically. ### Date subtraction (days between) @@ -129,7 +148,9 @@ mdvs search "researcher" --where "CAST('2032-01-01' AS DATE) - joined > 365" ### Null checks -`Date` and `DateTime` columns support standard null predicates, including for fields scoped to a subset of directories (rows outside the scope have null values for that column): +`Date` and `DateTime` columns support standard null predicates, including for +fields scoped to a subset of directories (rows outside the scope have null +values for that column): ```bash mdvs search "protocol" --where "last_reviewed IS NOT NULL" @@ -139,7 +160,8 @@ mdvs search "experiment" \ ### Combining with other filters -Date filters compose freely with the rest of the language — string compare, `IN`, `LIKE`, dotted-leaf access, array operations, and search ranking: +Date filters compose freely with the rest of the language — string compare, +`IN`, `LIKE`, dotted-leaf access, array operations, and search ranking: ```bash # Blog posts in 2031 H2 by specific authors @@ -153,7 +175,8 @@ mdvs search "experiment SPR" \ ## Array fields -Fields typed as `Array(String)` (like `tags`, `attendees`, `action_items`) support array functions. +Fields typed as `Array(String)` (like `tags`, `attendees`, `action_items`) +support array functions. ### Containment @@ -229,7 +252,9 @@ Searched "experiment" — 8 hits ... ``` -File paths are stored as relative paths (e.g., `projects/alpha/notes/experiment-1.md`). The **last component is the filename**, so you can match by directory, by filename, or by both: +File paths are stored as relative paths (e.g., +`projects/alpha/notes/experiment-1.md`). The **last component is the filename**, +so you can match by directory, by filename, or by both: ```bash # All blog posts (directory prefix) @@ -254,7 +279,8 @@ File paths are stored as relative paths (e.g., `projects/alpha/notes/experiment- ## Nested objects -Fields typed as Object (like `calibration` in `example_kb`) are stored as nested Struct columns. Access nested values with bracket notation: +Fields typed as Object (like `calibration` in `example_kb`) are stored as nested +Struct columns. Access nested values with bracket notation: ```bash mdvs search "sensor" --where "calibration['baseline']['wavelength'] > 600" @@ -280,7 +306,8 @@ Searched "sensor" — 2 hits ... ``` -The top-level field name (`calibration`) can be used bare. Only the nested access needs brackets: +The top-level field name (`calibration`) can be used bare. Only the nested +access needs brackets: ```bash # These are equivalent: @@ -290,7 +317,9 @@ The top-level field name (`calibration`) can be used bare. Only the nested acces ## Field names with special characters -Some field names need quoting in SQL. The [init](./commands/init.md), [update](./commands/update.md), and [info](./commands/info.md) commands show hints in their output when this applies. +Some field names need quoting in SQL. The [init](./commands/init.md), +[update](./commands/update.md), and [info](./commands/info.md) commands show +hints in their output when this applies. ### Spaces @@ -324,21 +353,27 @@ To include a literal single quote inside a string value, double it: mdvs search "query" --where "title = 'What''s New?'" ``` -mdvs validates quote balance before running the query. If you see "unmatched single quote", check that every `'` in a value is doubled. +mdvs validates quote balance before running the query. If you see "unmatched +single quote", check that every `'` in a value is doubled. ## Tips -- **Case sensitivity**: field names and string values are case-sensitive. Use `LOWER()` for case-insensitive matching: +- **Case sensitivity**: field names and string values are case-sensitive. Use + `LOWER()` for case-insensitive matching: + ```bash --where "LOWER(author) = 'giulia ferretti'" ``` - **LIKE patterns**: `%` matches any sequence, `_` matches a single character: + ```bash --where "title LIKE 'Project%'" # starts with "Project" --where "title LIKE '%sensor%'" # contains "sensor" ``` -- **NULL semantics**: comparisons against NULL always return false. Use `IS NULL` / `IS NOT NULL`, not `= NULL`. +- **NULL semantics**: comparisons against NULL always return false. Use + `IS NULL` / `IS NOT NULL`, not `= NULL`. -- **No aggregates in --where**: functions like `COUNT()` or `SUM()` don't work in `--where` — the filter applies per-file, not across results. +- **No aggregates in --where**: functions like `COUNT()` or `SUM()` don't work + in `--where` — the filter applies per-file, not across results. diff --git a/crates/mdvs/scaffolding/skill/SKILL.md b/crates/mdvs/scaffolding/skill/SKILL.md index 969e27c..1ed3499 100644 --- a/crates/mdvs/scaffolding/skill/SKILL.md +++ b/crates/mdvs/scaffolding/skill/SKILL.md @@ -11,7 +11,10 @@ description: >- # mdvs — Markdown Validation & Search -A CLI that treats a markdown directory as a database: schema inference, typed frontmatter validation, and semantic / full-text / hybrid search with SQL filters. Single binary, no external services. Full documentation at . +A CLI that treats a markdown directory as a database: schema inference, typed +frontmatter validation, and semantic / full-text / hybrid search with SQL +filters. Single binary, no external services. Full documentation at +. ## Usage @@ -56,11 +59,18 @@ mdvs scaffold snippet [--platform ] # AGENTS.md / CL mdvs scaffold hook --platform # PostToolUse hook config (JSON snippet) ``` -All commands take `--output `. **For agent context, pass `--output markdown` explicitly** or set `default_output_format = "markdown"` in `mdvs.toml`. `` defaults to `.` for every command. +All commands take `--output `. **For agent context, pass +`--output markdown` explicitly** or set `default_output_format = "markdown"` in +`mdvs.toml`. `` defaults to `.` for every command. ## What mdvs is for -mdvs treats a markdown directory as a database: it infers a typed schema from frontmatter, validates that schema on every edit, and searches the content semantically. The schema (`mdvs.toml`) is the source of truth; everything else (search index, validation reports) is derived. The schema is meant to **evolve with the KB** as conventions emerge — mdvs makes deviations visible without freezing them. +mdvs treats a markdown directory as a database: it infers a typed schema from +frontmatter, validates that schema on every edit, and searches the content +semantically. The schema (`mdvs.toml`) is the source of truth; everything else +(search index, validation reports) is derived. The schema is meant to **evolve +with the KB** as conventions emerge — mdvs makes deviations visible without +freezing them. ## What you must do when invoked @@ -68,21 +78,31 @@ mdvs treats a markdown directory as a database: it infers a typed schema from fr Look for `mdvs.toml` in the current working directory or any ancestor. -- **If found**, that directory (or its parent containing the file) is an mdvs vault. Treat any markdown file under it as living under that schema. -- **If not found** but the project has markdown files with frontmatter, mdvs is still relevant — propose bootstrapping with `mdvs init ` before reaching for Grep on the markdown. +- **If found**, that directory (or its parent containing the file) is an mdvs + vault. Treat any markdown file under it as living under that schema. +- **If not found** but the project has markdown files with frontmatter, mdvs is + still relevant — propose bootstrapping with `mdvs init ` before reaching + for Grep on the markdown. ### Step 2 — For any content lookup, use `mdvs search` first -When the user (or your own task) needs to find something in markdown content — a note about a topic, a project status, a person, an experiment, anything — **default to `mdvs search ""`, not Grep / Glob**. mdvs is built exactly for this: +When the user (or your own task) needs to find something in markdown content — a +note about a topic, a project status, a person, an experiment, anything — +**default to `mdvs search ""`, not Grep / Glob**. mdvs is built exactly +for this: -- `--mode hybrid` (default) — semantic + BM25 reranked; best general-purpose mode -- `--mode semantic` — vector only; best for "find notes about X" where X is a concept, not a phrase +- `--mode hybrid` (default) — semantic + BM25 reranked; best general-purpose + mode +- `--mode semantic` — vector only; best for "find notes about X" where X is a + concept, not a phrase - `--mode fulltext` — BM25 only; for known literal phrases - `--where "field = 'value'"` — filter by frontmatter (SQL syntax) - `--limit N` — cap result count (default 10) - `-v` — show the best matching chunk text per result -Only fall back to Grep when (a) you need a literal substring match and already know which file to look in, or (b) `mdvs search` returns no results and you suspect missing content rather than missing relevance. +Only fall back to Grep when (a) you need a literal substring match and already +know which file to look in, or (b) `mdvs search` returns no results and you +suspect missing content rather than missing relevance. ### Step 3 — For frontmatter changes, check then evolve @@ -90,57 +110,88 @@ After any edit that touches frontmatter (yours or the user's): 1. Run `mdvs check` to validate against the schema. 2. If **violations** appear: see Step 4 (the schema-evolution loop). -3. If **new fields** appear (present in files but not in `mdvs.toml`): they show up as informational, not violations. Run `mdvs update` to add them, or `mdvs update reinfer ` to refresh a specific field's constraints. +3. If **new fields** appear (present in files but not in `mdvs.toml`): they show + up as informational, not violations. Run `mdvs update` to add them, or + `mdvs update reinfer ` to refresh a specific field's constraints. ### Step 4 — When a hook surfaces a violation, follow the schema-evolution loop -If a markdown block lands in your context from a `PostToolUse` hook listing `MissingRequired` / `WrongType` / `Disallowed` / `InvalidCategory` / `OutOfRange` violations, **that's mdvs talking to you via the validation hook**. Claude Code surfaces it under `additionalContext`; other harnesses use their own channel (the wiring is harness-specific). The hook is non-blocking by design: the edit already landed; the warning is for you to act on next. +If a markdown block lands in your context from a `PostToolUse` hook listing +`MissingRequired` / `WrongType` / `Disallowed` / `InvalidCategory` / +`OutOfRange` violations, **that's mdvs talking to you via the validation hook**. +Claude Code surfaces it under `additionalContext`; other harnesses use their own +channel (the wiring is harness-specific). The hook is non-blocking by design: +the edit already landed; the warning is for you to act on next. Your job: -1. Read the violation block: which file, which field, which rule, expected vs actual. -2. Decide: **mistake** (typo, wrong type by accident, dropped required field) or **intentional** (KB is evolving, category needs a new variant, field shifting type)? -3. **Mistake** → fix the file in the next turn. Acknowledge briefly so the user knows the loop is working. -4. **Intentional** → surface the deviation to the user and propose updating `mdvs.toml` (`mdvs update`, `mdvs update reinfer --with=`, or a manual edit). **Do not silently fix the file.** The user decides whether the schema or the file is the source of truth. - -A worked example of the intentional path is in the [Examples](#example--responding-to-a-hook-delivered-violation) section. +1. Read the violation block: which file, which field, which rule, expected vs + actual. +2. Decide: **mistake** (typo, wrong type by accident, dropped required field) or + **intentional** (KB is evolving, category needs a new variant, field shifting + type)? +3. **Mistake** → fix the file in the next turn. Acknowledge briefly so the user + knows the loop is working. +4. **Intentional** → surface the deviation to the user and propose updating + `mdvs.toml` (`mdvs update`, `mdvs update reinfer --with=`, or a + manual edit). **Do not silently fix the file.** The user decides whether the + schema or the file is the source of truth. + +A worked example of the intentional path is in the +[Examples](#example--responding-to-a-hook-delivered-violation) section. ## Rules -- **Prefer `mdvs search` over Grep / Glob in any markdown corpus.** Inside an mdvs vault, the default content-search tool is mdvs. -- **The validation hook is a warning, not a block.** Treat violations as a prompt for discussion, not as edits to be reverted. -- **Never silently fix an intentional deviation.** Propose a schema update; let the user decide. -- **The schema is meant to evolve.** A schema that never changes is a schema that's wrong. Enforcement follows the KB's shape; it does not freeze it. +- **Prefer `mdvs search` over Grep / Glob in any markdown corpus.** Inside an + mdvs vault, the default content-search tool is mdvs. +- **The validation hook is a warning, not a block.** Treat violations as a + prompt for discussion, not as edits to be reverted. +- **Never silently fix an intentional deviation.** Propose a schema update; let + the user decide. +- **The schema is meant to evolve.** A schema that never changes is a schema + that's wrong. Enforcement follows the KB's shape; it does not freeze it. - **`build` always runs `check` first.** Validation gates the index build. -- **`mdvs.toml` is the only source of truth.** `.mdvs/` is derived — gitignored, recreatable with `mdvs build`. +- **`mdvs.toml` is the only source of truth.** `.mdvs/` is derived — gitignored, + recreatable with `mdvs build`. - **There is no lock file.** Schema changes flow through `mdvs.toml` only. -- **Frontmatter formats are auto-detected** per file (YAML / TOML / JSON). A single vault can mix all three. -- **Use `--output markdown` for any output you intend to read.** It's the format LLMs parse most fluently, and the format the validation hook surfaces back through the harness's model-context channel. +- **Frontmatter formats are auto-detected** per file (YAML / TOML / JSON). A + single vault can mix all three. +- **Use `--output markdown` for any output you intend to read.** It's the format + LLMs parse most fluently, and the format the validation hook surfaces back + through the harness's model-context channel. ## Two layers mdvs has two independent layers: -1. **Validation** (`init`, `check`, `update`) — works immediately, no model download, no build step. Reads markdown and validates frontmatter against `mdvs.toml`. -2. **Search** (`build`, `search`) — downloads an embedding model, chunks markdown content, builds a local LanceDB index in `.mdvs/`. +1. **Validation** (`init`, `check`, `update`) — works immediately, no model + download, no build step. Reads markdown and validates frontmatter against + `mdvs.toml`. +2. **Search** (`build`, `search`) — downloads an embedding model, chunks + markdown content, builds a local LanceDB index in `.mdvs/`. Validation stands alone. You never need to build an index just to validate. ## Key files -- **`mdvs.toml`** — schema config, committed to version control. Source of truth for field types, allowed/required paths, constraints. -- **`.mdvs/`** — build artifacts (the Lance dataset under `index.lance/` plus a cached model). To be gitignored. Recreatable with `mdvs build`. Never edit directly. +- **`mdvs.toml`** — schema config, committed to version control. Source of truth + for field types, allowed/required paths, constraints. +- **`.mdvs/`** — build artifacts (the Lance dataset under `index.lance/` plus a + cached model). To be gitignored. Recreatable with `mdvs build`. Never edit + directly. ## Command reference ### `mdvs init` -Scans markdown files, infers a typed schema from frontmatter, writes `mdvs.toml`. +Scans markdown files, infers a typed schema from frontmatter, writes +`mdvs.toml`. - `--force` — overwrite an existing `mdvs.toml` (deletes `.mdvs/` too) - `--dry-run` — show what would be inferred without writing - `--ignore-bare-files` — exclude files that have no frontmatter -- `--from-jsonschema PATH` — import schema from an external JSON Schema 2020-12 document. Round-trips with `mdvs export-jsonschema`. +- `--from-jsonschema PATH` — import schema from an external JSON Schema 2020-12 + document. Round-trips with `mdvs export-jsonschema`. Use `init --force` to start over. Use `update` to incrementally add new fields. @@ -154,15 +205,18 @@ Validates all frontmatter against `mdvs.toml`. Reports violation kinds: - **`InvalidCategory`** — value is not in the declared category list - **`OutOfRange`** — numeric value outside declared `min`/`max` -New fields (in files but not in `mdvs.toml`) are reported separately as informational — no non-zero exit. Run `update` to add them. +New fields (in files but not in `mdvs.toml`) are reported separately as +informational — no non-zero exit. Run `update` to add them. - `--jsonschema PATH` — override `[fields]` in `mdvs.toml` for this run -Violation output is deterministic: sorted by `(field, kind, rule)` and `path` within. +Violation output is deterministic: sorted by `(field, kind, rule)` and `path` +within. ### `mdvs update` -Re-scans files; adds newly discovered fields to `mdvs.toml`. Doesn't remove or change existing fields by default. +Re-scans files; adds newly discovered fields to `mdvs.toml`. Doesn't remove or +change existing fields by default. - `mdvs update` — detect and add new fields - `mdvs update reinfer ` — re-infer type and constraints @@ -171,27 +225,37 @@ Re-scans files; adds newly discovered fields to `mdvs.toml`. Doesn't remove or c - `mdvs update reinfer --with=range` — infer min/max - `mdvs update reinfer --with=none` — strip all constraints -Use `reinfer` when a field's type has changed or you want to refresh its constraints. `--with` requires a named field. +Use `reinfer` when a field's type has changed or you want to refresh its +constraints. `--with` requires a named field. ### `mdvs build` -Validates, then chunks markdown, generates embeddings, writes the Lance dataset to `.mdvs/`. +Validates, then chunks markdown, generates embeddings, writes the Lance dataset +to `.mdvs/`. - `--force` — full rebuild (ignore incremental cache) - Incremental by default — only re-embeds new or edited files - Aborts if `check` finds violations -First build downloads the default embedding model `minishlab/potion-multilingual-128M` (~480 MB, 101 languages). Subsequent builds reuse it. +First build downloads the default embedding model +`minishlab/potion-multilingual-128M` (~480 MB, 101 languages). Subsequent builds +reuse it. ### `mdvs search` -Searches the indexed notes — semantic (vector), full-text (BM25), or hybrid (RRF reranker). Auto-builds the index if needed. +Searches the indexed notes — semantic (vector), full-text (BM25), or hybrid (RRF +reranker). Auto-builds the index if needed. ```bash mdvs search "" [path] [--mode ] [--where ""] [--limit N] [-v] ``` -`--where` operates on **any column** in the Lance index: frontmatter fields (auto-discovered from `mdvs.toml`, referenced by bare name) and the always-present `filepath` column. Field names with spaces need double-quote escaping: `--where "\"lab section\" = 'Photonics'"`. Filtering on `Array(Float)` fields is rejected up front (Lance can't safely decode them); store as parallel scalar arrays. +`--where` operates on **any column** in the Lance index: frontmatter fields +(auto-discovered from `mdvs.toml`, referenced by bare name) and the +always-present `filepath` column. Field names with spaces need double-quote +escaping: `--where "\"lab section\" = 'Photonics'"`. Filtering on `Array(Float)` +fields is rejected up front (Lance can't safely decode them); store as parallel +scalar arrays. #### Scalar frontmatter — equality, inequality, comparison @@ -224,7 +288,9 @@ mdvs search "" [path] [--mode ] [--where ""] [--limit N] [-v] #### Array fields — auto-rewritten to `array_has(...)` -The `=` / `!=` / `IN` / `NOT IN` operators against an array field auto-rewrite to `array_has(...)` so element-containment "just works". The search output shows the rewrite as a one-line `Note` at the top. +The `=` / `!=` / `IN` / `NOT IN` operators against an array field auto-rewrite +to `array_has(...)` so element-containment "just works". The search output shows +the rewrite as a one-line `Note` at the top. ```bash --where "tags = 'rust'" # has 'rust' as one of its tags @@ -238,7 +304,8 @@ The `=` / `!=` / `IN` / `NOT IN` operators against an array field auto-rewrite t #### Date fields -Date literals use the `date '...'` keyword form. RFC 3339 datetimes use `timestamp '...'`. +Date literals use the `date '...'` keyword form. RFC 3339 datetimes use +`timestamp '...'`. ```bash --where "published > date '2024-01-01'" @@ -248,7 +315,9 @@ Date literals use the `date '...'` keyword form. RFC 3339 datetimes use `timesta #### Path filtering — the always-present `filepath` column -The `filepath` column stores the path relative to the project root — the **last component is the filename** (e.g. `articles/long-essay-2024.md` → filename is `long-essay-2024.md`). +The `filepath` column stores the path relative to the project root — the **last +component is the filename** (e.g. `articles/long-essay-2024.md` → filename is +`long-essay-2024.md`). ```bash --where "filepath LIKE 'articles/%'" # everything under articles/ @@ -269,7 +338,8 @@ The `filepath` column stores the path relative to the project root — the **las #### Functions -Most DataFusion scalar functions work — handy when the raw value doesn't quite match. +Most DataFusion scalar functions work — handy when the raw value doesn't quite +match. ```bash --where "length(title) > 50" # long titles @@ -277,11 +347,14 @@ Most DataFusion scalar functions work — handy when the raw value doesn't quite --where "year + 1 > 2025" # arithmetic ``` -Other internal columns exist (`start_line`, `end_line`, `built_at`, `chunk_text`) but are rarely useful for filtering — semantic / fulltext search handles those concerns better. +Other internal columns exist (`start_line`, `end_line`, `built_at`, +`chunk_text`) but are rarely useful for filtering — semantic / fulltext search +handles those concerns better. ### `mdvs info` -Shows current config and index status: scan settings, field definitions, build metadata (model, chunk size, file counts). `-v` for full field detail. +Shows current config and index status: scan settings, field definitions, build +metadata (model, chunk size, file counts). `-v` for full field detail. ### `mdvs clean` @@ -289,38 +362,58 @@ Deletes `.mdvs/`. Doesn't touch `mdvs.toml`. ### `mdvs export-jsonschema` -Translates `[fields]` into a canonical JSON Schema 2020-12 document. Useful for sharing with other tools, or round-tripping through `mdvs init --from-jsonschema`. +Translates `[fields]` into a canonical JSON Schema 2020-12 document. Useful for +sharing with other tools, or round-tripping through +`mdvs init --from-jsonschema`. - `--format json|toml` — output format (default `json`) - `--output-file FILE` — write to file instead of stdout ### `mdvs scaffold` -Emits the artifacts that integrate mdvs with an agent harness (Claude Code, Codex, OpenCode, Cursor, Antigravity). Each subcommand prints to stdout; pipe it into the right location for your harness. - -- `mdvs scaffold skill [--platform ]` — this skill file. Default destination: `.agents/skills/mdvs/SKILL.md` for Codex / OpenCode / Cursor / Antigravity; `.claude/skills/mdvs/SKILL.md` for Claude Code. -- `mdvs scaffold snippet [--platform ]` — the project-rules snippet for `AGENTS.md` / `CLAUDE.md` / `.cursor/rules/mdvs.mdc`. -- `mdvs scaffold hook --platform claude-code` — the `PostToolUse` hook config (a JSON snippet to merge into `.claude/settings.json`). The emitted snippet's `command:` fields call `mdvs hook handle` directly — no shell scripts, no `jq` dependency. **Currently the only verified hook integration.** The other platforms refuse with a pointer to their per-platform mdbook page; wiring `mdvs hook handle` into Codex / Cursor / OpenCode / Antigravity is possible by following each harness's own hooks documentation. +Emits the artifacts that integrate mdvs with an agent harness (Claude Code, +Codex, OpenCode, Cursor, Antigravity). Each subcommand prints to stdout; pipe it +into the right location for your harness. + +- `mdvs scaffold skill [--platform ]` — this skill file. Default + destination: `.agents/skills/mdvs/SKILL.md` for Codex / OpenCode / Cursor / + Antigravity; `.claude/skills/mdvs/SKILL.md` for Claude Code. +- `mdvs scaffold snippet [--platform ]` — the project-rules snippet for + `AGENTS.md` / `CLAUDE.md` / `.cursor/rules/mdvs.mdc`. +- `mdvs scaffold hook --platform claude-code` — the `PostToolUse` hook config (a + JSON snippet to merge into `.claude/settings.json`). The emitted snippet's + `command:` fields call `mdvs hook handle` directly — no shell scripts, no `jq` + dependency. **Currently the only verified hook integration.** The other + platforms refuse with a pointer to their per-platform mdbook page; wiring + `mdvs hook handle` into Codex / Cursor / OpenCode / Antigravity is possible by + following each harness's own hooks documentation. ## Agent-harness integration -mdvs ships as a CLI; integrating it with an agent harness is wiring rather than installing. Three artifacts cover the three integration points: +mdvs ships as a CLI; integrating it with an agent harness is wiring rather than +installing. Three artifacts cover the three integration points: -| Artifact | Purpose | Coverage | -|---|---|---| -| **Skill file** (`SKILL.md`, this file) | Activated by harnesses implementing the [Agent Skills open standard](https://agentskills.io). Loaded on demand; agent reads procedure + reference. | Works in any harness that loads `.md` skills (Claude Code, Codex, Cursor, OpenCode, Antigravity, …). | -| **Project-rules snippet** | Always-on text in `AGENTS.md` / `CLAUDE.md` / `.cursor/rules/mdvs.mdc`. Short — names the KB, the search-vs-Grep preference, and the warning-loop rule. | Works in any harness that reads `AGENTS.md` / `CLAUDE.md` / `.cursor/rules`. | -| **`PostToolUse` hook** | Calls `mdvs hook handle` after every Edit / Write on a markdown file inside the vault. mdvs walks up to find `mdvs.toml`, runs `check`, and surfaces violations to you as **non-blocking** model-context. | **Shipped for Claude Code only.** Other harnesses: follow their documented hook system to wire `mdvs hook handle` in — see . | +| Artifact | Purpose | Coverage | +| -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Skill file** (`SKILL.md`, this file) | Activated by harnesses implementing the [Agent Skills open standard](https://agentskills.io). Loaded on demand; agent reads procedure + reference. | Works in any harness that loads `.md` skills (Claude Code, Codex, Cursor, OpenCode, Antigravity, …). | +| **Project-rules snippet** | Always-on text in `AGENTS.md` / `CLAUDE.md` / `.cursor/rules/mdvs.mdc`. Short — names the KB, the search-vs-Grep preference, and the warning-loop rule. | Works in any harness that reads `AGENTS.md` / `CLAUDE.md` / `.cursor/rules`. | +| **`PostToolUse` hook** | Calls `mdvs hook handle` after every Edit / Write on a markdown file inside the vault. mdvs walks up to find `mdvs.toml`, runs `check`, and surfaces violations to you as **non-blocking** model-context. | **Shipped for Claude Code only.** Other harnesses: follow their documented hook system to wire `mdvs hook handle` in — see . | ## Output format Three formats; default is `pretty`. - `--output pretty` — box-drawing tables for terminal display. Adapts to width. -- `--output markdown` — GFM tables and `##` headers. **Best for agent consumption** and the format the validation hook surfaces back through the harness's model-context channel. -- `--output json` — structured JSON for programmatic extraction (pipe through `jq` or any JSON tool). +- `--output markdown` — GFM tables and `##` headers. **Best for agent + consumption** and the format the validation hook surfaces back through the + harness's model-context channel. +- `--output json` — structured JSON for programmatic extraction (pipe through + `jq` or any JSON tool). -Priority chain when `--output` is omitted: CLI flag > `default_output_format` in `mdvs.toml` > hard default (`pretty`). Same command always produces the same output regardless of TTY state. `-v` (verbose) adds per-step pipeline output with timings. +Priority chain when `--output` is omitted: CLI flag > `default_output_format` in +`mdvs.toml` > hard default (`pretty`). Same command always produces the same +output regardless of TTY state. `-v` (verbose) adds per-step pipeline output +with timings. ## Exit codes @@ -328,20 +421,55 @@ Priority chain when `--output` is omitted: CLI flag > `default_output_format` in - **1** — violations found (`check` and `build`) - **2** — error (bad config, missing files, model mismatch) -Hook scripts use `|| true` to mask the exit-1 from `check` — the hook is intentionally non-blocking. Don't change that contract. +Hook scripts use `|| true` to mask the exit-1 from `check` — the hook is +intentionally non-blocking. Don't change that contract. ## Things to know -- Field types inferred automatically: `String`, `Integer`, `Float`, `Boolean`, `Date` (`YYYY-MM-DD`), `DateTime` (RFC 3339 with mandatory timezone), and `Array()` for any scalar type. The on-disk grammar is `Scalar | Array(Scalar)` only. Mixed scalar types widen (`Integer + String → String`). -- **Nested frontmatter uses dotted-name leaves.** `calibration.baseline.wavelength: 850.0` becomes a `[[fields.field]]` named `"calibration.baseline.wavelength"` of type `Float`. Top-level `Object` is rejected; nested `Array(Object{...})` is also rejected — represent arrays of structured items as parallel scalar arrays (e.g. `measurement_timestamps: Array(String)` + `measurement_values: Array(Float)`). SQL filters use dot notation: `--where "calibration.baseline.wavelength > 800"`. -- **Preprocessors opt into widening.** Each field carries a `preprocess` array. Built-ins: `coerce_to_string` (accepts non-string scalars on a `String` field) and `widen_int_to_float` (accepts integers on a `Float` field). Inference auto-populates these when widening was observed. `preprocess = []` means strict. -- Categorical detection is automatic for low-cardinality repeated values. Out-of-category values → `InvalidCategory`. -- Constraint kinds: `categories` (closed-set enum, mutually exclusive with everything else), `min`/`max` (numeric range), `min_length`/`max_length` (string and array length), `pattern` (regex on strings). Range / length / pattern are not auto-inferred; add manually or via `update reinfer --with=range`. -- `init --force` rewrites the config from scratch. `update` preserves existing config and only adds new fields. `update reinfer` re-infers specific fields. -- Model identity is tracked: changing the model in `mdvs.toml` requires `build --force` to confirm a full re-embed. -- `check` auto-runs `update` first by default (unless `--no-update`, or `--jsonschema` is given, or `[check].auto_update = false` is set in `mdvs.toml`). -- `search` auto-runs `update` and `build` if needed (unless `--no-update` / `--no-build`). -- **`.mdvsignore` and `.gitignore` are both honored** when scanning. mdvs reuses the [`ignore`](https://crates.io/crates/ignore) crate (same matcher git uses), so the per-directory `.gitignore` rules you already have apply automatically. For mdvs-only exclusions (e.g. "don't index this draft directory, but keep it tracked in git"), drop a `.mdvsignore` next to the files using the same syntax as `.gitignore`. Both apply to `init` / `update` / `check` / `build` / `search`. To stop honoring `.gitignore` (for example, to index files that are deliberately untracked but should still be searchable), set `[scan].skip_gitignore = true` in `mdvs.toml`. Hidden files and dotfiles are NOT skipped — only `.md` / `.markdown` extensions are scanned to begin with. +- Field types inferred automatically: `String`, `Integer`, `Float`, `Boolean`, + `Date` (`YYYY-MM-DD`), `DateTime` (RFC 3339 with mandatory timezone), and + `Array()` for any scalar type. The on-disk grammar is + `Scalar | Array(Scalar)` only. Mixed scalar types widen + (`Integer + String → String`). +- **Nested frontmatter uses dotted-name leaves.** + `calibration.baseline.wavelength: 850.0` becomes a `[[fields.field]]` named + `"calibration.baseline.wavelength"` of type `Float`. Top-level `Object` is + rejected; nested `Array(Object{...})` is also rejected — represent arrays of + structured items as parallel scalar arrays (e.g. + `measurement_timestamps: Array(String)` + `measurement_values: Array(Float)`). + SQL filters use dot notation: + `--where "calibration.baseline.wavelength > 800"`. +- **Preprocessors opt into widening.** Each field carries a `preprocess` array. + Built-ins: `coerce_to_string` (accepts non-string scalars on a `String` field) + and `widen_int_to_float` (accepts integers on a `Float` field). Inference + auto-populates these when widening was observed. `preprocess = []` means + strict. +- Categorical detection is automatic for low-cardinality repeated values. + Out-of-category values → `InvalidCategory`. +- Constraint kinds: `categories` (closed-set enum, mutually exclusive with + everything else), `min`/`max` (numeric range), `min_length`/`max_length` + (string and array length), `pattern` (regex on strings). Range / length / + pattern are not auto-inferred; add manually or via + `update reinfer --with=range`. +- `init --force` rewrites the config from scratch. `update` preserves existing + config and only adds new fields. `update reinfer` re-infers specific fields. +- Model identity is tracked: changing the model in `mdvs.toml` requires + `build --force` to confirm a full re-embed. +- `check` auto-runs `update` first by default (unless `--no-update`, or + `--jsonschema` is given, or `[check].auto_update = false` is set in + `mdvs.toml`). +- `search` auto-runs `update` and `build` if needed (unless `--no-update` / + `--no-build`). +- **`.mdvsignore` and `.gitignore` are both honored** when scanning. mdvs reuses + the [`ignore`](https://crates.io/crates/ignore) crate (same matcher git uses), + so the per-directory `.gitignore` rules you already have apply automatically. + For mdvs-only exclusions (e.g. "don't index this draft directory, but keep it + tracked in git"), drop a `.mdvsignore` next to the files using the same syntax + as `.gitignore`. Both apply to `init` / `update` / `check` / `build` / + `search`. To stop honoring `.gitignore` (for example, to index files that are + deliberately untracked but should still be searchable), set + `[scan].skip_gitignore = true` in `mdvs.toml`. Hidden files and dotfiles are + NOT skipped — only `.md` / `.markdown` extensions are scanned to begin with. ## Examples @@ -373,15 +501,23 @@ mdvs check ``` Resolution per kind: -- **MissingRequired** — add the field, or remove the path from `required` in `mdvs.toml` -- **WrongType** — fix the value, or `mdvs update reinfer ` if the type should change -- **InvalidCategory** — fix the value, or `mdvs update reinfer --with=categorical` to update the list -- **OutOfRange** — fix the value, or `mdvs update reinfer --with=range` to widen bounds -- **Disallowed** — remove the field from that file, or widen `allowed` globs in `mdvs.toml` + +- **MissingRequired** — add the field, or remove the path from `required` in + `mdvs.toml` +- **WrongType** — fix the value, or `mdvs update reinfer ` if the type + should change +- **InvalidCategory** — fix the value, or + `mdvs update reinfer --with=categorical` to update the list +- **OutOfRange** — fix the value, or `mdvs update reinfer --with=range` + to widen bounds +- **Disallowed** — remove the field from that file, or widen `allowed` globs in + `mdvs.toml` ### Example — responding to a hook-delivered violation -You wrote `status: in_review` to `kb/projects/alpha/sprint-12.md`. The next thing in your context is a hook-surfaced violation block (the exact channel name varies per harness; the content is the same): +You wrote `status: in_review` to `kb/projects/alpha/sprint-12.md`. The next +thing in your context is a hook-surfaced violation block (the exact channel name +varies per harness; the content is the same): ``` ## Violations @@ -393,9 +529,17 @@ You wrote `status: in_review` to `kb/projects/alpha/sprint-12.md`. The next thin Your response: -> Just wrote `status: in_review` on `sprint-12.md`. mdvs's schema only allows `draft | published | archived` for `status`. Looking back, I've seen the `in_review` state come up in two other recent edits — this might be a real new lifecycle stage rather than a typo. Want me to add `in_review` to the schema (`mdvs update reinfer status --with=categorical` after updating the file with the new value), or should I change the value on `sprint-12.md` to one of the existing categories? +> Just wrote `status: in_review` on `sprint-12.md`. mdvs's schema only allows +> `draft | published | archived` for `status`. Looking back, I've seen the +> `in_review` state come up in two other recent edits — this might be a real new +> lifecycle stage rather than a typo. Want me to add `in_review` to the schema +> (`mdvs update reinfer status --with=categorical` after updating the file with +> the new value), or should I change the value on `sprint-12.md` to one of the +> existing categories? -Note what you did NOT do: silently `mdvs update reinfer status --with=categorical` to make the warning go away. The user decides whether the schema or the file is the source of truth. +Note what you did NOT do: silently +`mdvs update reinfer status --with=categorical` to make the warning go away. The +user decides whether the schema or the file is the source of truth. ### Wiring mdvs into Claude Code (end-to-end) @@ -405,22 +549,29 @@ mdvs scaffold snippet >> CLAUDE.md # the always-on rules blo mdvs scaffold hook --platform claude-code # PostToolUse hook config — read stderr for the destination ``` -`mdvs scaffold hook` prints a JSON snippet to merge into `.claude/settings.json`. No shell scripts; the `command:` fields call `mdvs hook handle` directly. +`mdvs scaffold hook` prints a JSON snippet to merge into +`.claude/settings.json`. No shell scripts; the `command:` fields call +`mdvs hook handle` directly. ### Wiring mdvs into other harnesses (skill + snippet) -For Codex, Cursor, OpenCode, or Antigravity, install the skill and snippet — the hook half isn't shipped: +For Codex, Cursor, OpenCode, or Antigravity, install the skill and snippet — the +hook half isn't shipped: ```bash mdvs scaffold skill --platform > /mdvs/SKILL.md mdvs scaffold snippet --platform >> ``` -`mdvs scaffold hook --platform ` for these harnesses refuses with a pointer at , which describes how to wire `mdvs hook handle` into each harness's own hook config using that harness's documentation. +`mdvs scaffold hook --platform ` for these harnesses refuses with a +pointer at , which +describes how to wire `mdvs hook handle` into each harness's own hook config +using that harness's documentation. ### Searching with filters -Full reference for `--where` is in the [search reference section](#mdvs-search) above. A few realistic shapes: +Full reference for `--where` is in the [search reference section](#mdvs-search) +above. A few realistic shapes: ```bash # Recent articles by a specific author @@ -444,20 +595,29 @@ mdvs search "calibration" -v ### Edge cases -- **Files without frontmatter (bare files):** `init` includes them by default. Use `--ignore-bare-files` or set `include_bare_files = false` in `[scan]` to exclude. -- **Null values:** `nullable = true` accepts null. Null skips type and category checks. A `required` + `nullable` field passes with `key: null` — fails only if the key is entirely absent. -- **Mixed-type fields:** widen to `String`. `1` becomes `"1"`. Intentional, not data loss. -- **Special characters in field names:** TOML handles quoting in `mdvs.toml`. In `--where`, wrap with double quotes: `--where "\"author's note\" IS NOT NULL"`. -- **Hook fires on non-vault edits:** if no `mdvs.toml` is reachable upward from the edited file, the hook exits silently. No false positives. -- **Hook stays silent on a bad edit:** check (in order) — harness matcher pattern, file extension (`.md`), `mdvs` on PATH for the harness's hook subprocess, symlinks outside the vault. +- **Files without frontmatter (bare files):** `init` includes them by default. + Use `--ignore-bare-files` or set `include_bare_files = false` in `[scan]` to + exclude. +- **Null values:** `nullable = true` accepts null. Null skips type and category + checks. A `required` + `nullable` field passes with `key: null` — fails only + if the key is entirely absent. +- **Mixed-type fields:** widen to `String`. `1` becomes `"1"`. Intentional, not + data loss. +- **Special characters in field names:** TOML handles quoting in `mdvs.toml`. In + `--where`, wrap with double quotes: `--where "\"author's note\" IS NOT NULL"`. +- **Hook fires on non-vault edits:** if no `mdvs.toml` is reachable upward from + the edited file, the hook exits silently. No false positives. +- **Hook stays silent on a bad edit:** check (in order) — harness matcher + pattern, file extension (`.md`), `mdvs` on PATH for the harness's hook + subprocess, symlinks outside the vault. ## Common errors -| Error | Cause | Fix | -|---|---|---| -| `mdvs.toml already exists` | Running `init` twice | `init --force` or `update` | -| `no markdown files found` | Wrong path or glob | Check path + `[scan].glob` in config | -| `model mismatch` | Config model differs from index | `build --force` to re-embed | -| `field 'X' is not in mdvs.toml` | `reinfer` on unknown field | Check spelling, or `update` first to add it | -| Violations on `check` | Frontmatter doesn't match schema | Read the list, fix files or evolve the schema (Step 4) | -| Hook stays silent on a bad edit | See Edge cases above | Check matcher, extension, PATH, vault location | +| Error | Cause | Fix | +| ------------------------------- | -------------------------------- | ------------------------------------------------------ | +| `mdvs.toml already exists` | Running `init` twice | `init --force` or `update` | +| `no markdown files found` | Wrong path or glob | Check path + `[scan].glob` in config | +| `model mismatch` | Config model differs from index | `build --force` to re-embed | +| `field 'X' is not in mdvs.toml` | `reinfer` on unknown field | Check spelling, or `update` first to add it | +| Violations on `check` | Frontmatter doesn't match schema | Read the list, fix files or evolve the schema (Step 4) | +| Hook stays silent on a bad edit | See Edge cases above | Check matcher, extension, PATH, vault location | diff --git a/crates/mdvs/scaffolding/snippet/agents-md.md b/crates/mdvs/scaffolding/snippet/agents-md.md index b267524..fa34da6 100644 --- a/crates/mdvs/scaffolding/snippet/agents-md.md +++ b/crates/mdvs/scaffolding/snippet/agents-md.md @@ -3,14 +3,31 @@ ## mdvs knowledge base -This project uses [mdvs](https://github.com/edochi/mdvs) to manage a markdown knowledge base with a schema defined in `mdvs.toml`. When working with files in this KB: +This project uses [mdvs](https://github.com/edochi/mdvs) to manage a markdown +knowledge base with a schema defined in `mdvs.toml`. When working with files in +this KB: -- **Prefer `mdvs search ""` over `Grep` / `Glob` for semantic lookups.** mdvs runs semantic / hybrid / SQL-filtered search over the KB — usually a better fit than literal pattern matching for finding notes by meaning. Filter by frontmatter with `--where "field = 'value'"` and pick the mode with `--mode semantic|fulltext|hybrid`. -- **`mdvs check` validates frontmatter against the schema.** A `PostToolUse` hook may surface violations to you automatically as a markdown block under `additionalContext` in the hook output. Treat that block as **a warning, not a block** — the agent's edit already landed; the warning is for you to act on next. +- **Prefer `mdvs search ""` over `Grep` / `Glob` for semantic lookups.** + mdvs runs semantic / hybrid / SQL-filtered search over the KB — usually a + better fit than literal pattern matching for finding notes by meaning. Filter + by frontmatter with `--where "field = 'value'"` and pick the mode with + `--mode semantic|fulltext|hybrid`. +- **`mdvs check` validates frontmatter against the schema.** A `PostToolUse` + hook may surface violations to you automatically as a markdown block under + `additionalContext` in the hook output. Treat that block as **a warning, not a + block** — the agent's edit already landed; the warning is for you to act on + next. - **When a violation reaches you, decide:** - 1. **Mistake** (a typo, the wrong type by accident, a dropped required field) → fix the file in the next turn. - 2. **Intentional deviation** (the KB is evolving, a category genuinely needs a new variant, a field is shifting type) → surface the deviation to the user and propose updating `mdvs.toml` to absorb the change. **Do not silently fix the file**; the user decides whether the schema or the file is wrong. + 1. **Mistake** (a typo, the wrong type by accident, a dropped required field) + → fix the file in the next turn. + 2. **Intentional deviation** (the KB is evolving, a category genuinely needs a + new variant, a field is shifting type) → surface the deviation to the user + and propose updating `mdvs.toml` to absorb the change. **Do not silently + fix the file**; the user decides whether the schema or the file is wrong. -The schema is meant to evolve with the KB. Enforcement follows the KB's shape, it does not freeze it. Whenever you propose a schema update, also explain *why* — which file(s) drove the change and what convention the user might want to standardise on. +The schema is meant to evolve with the KB. Enforcement follows the KB's shape, +it does not freeze it. Whenever you propose a schema update, also explain _why_ +— which file(s) drove the change and what convention the user might want to +standardise on. Full mdvs documentation: . diff --git a/crates/tomljson/README.md b/crates/tomljson/README.md index aae9d73..e803bd3 100644 --- a/crates/tomljson/README.md +++ b/crates/tomljson/README.md @@ -4,15 +4,28 @@ Lossless TOML ↔ JSON translation in pure Rust. ## What it does -Encode `serde_json::Value` to TOML and back, handling the impedance gaps where TOML's data model can't natively represent something JSON can: - -- **Null** — TOML has no null type. JSON `null` is encoded as a string placeholder (default `"__null__"`, configurable). Decoding substitutes the placeholder back to `null`. -- **Top-level non-table values** — TOML documents must have a table at the root. Non-table JSON values (booleans, scalars, arrays) are wrapped under a configurable root key (default `"__root__"`) on encode and unwrapped on decode. -- **Integer range** — TOML integers are signed 64-bit. Encoding rejects values larger than `i64::MAX`. -- **Float fidelity** — `f64` values round-trip via `f64::to_string` (Ryū-shortest representation). NaN and ±∞ error explicitly on decode (JSON's number model can't represent them). -- **Datetime canonicalization** — all four TOML datetime variants (`Date`, `Time`, `LocalDateTime`, `OffsetDateTime`) decode to JSON strings in canonical RFC 3339 form. - -The motivating use case is JSON Schema 2020-12 documents authored as TOML, but `tomljson` is application-agnostic — anyone moving JSON-shaped data through TOML can use it. +Encode `serde_json::Value` to TOML and back, handling the impedance gaps where +TOML's data model can't natively represent something JSON can: + +- **Null** — TOML has no null type. JSON `null` is encoded as a string + placeholder (default `"__null__"`, configurable). Decoding substitutes the + placeholder back to `null`. +- **Top-level non-table values** — TOML documents must have a table at the root. + Non-table JSON values (booleans, scalars, arrays) are wrapped under a + configurable root key (default `"__root__"`) on encode and unwrapped on + decode. +- **Integer range** — TOML integers are signed 64-bit. Encoding rejects values + larger than `i64::MAX`. +- **Float fidelity** — `f64` values round-trip via `f64::to_string` + (Ryū-shortest representation). NaN and ±∞ error explicitly on decode (JSON's + number model can't represent them). +- **Datetime canonicalization** — all four TOML datetime variants (`Date`, + `Time`, `LocalDateTime`, `OffsetDateTime`) decode to JSON strings in canonical + RFC 3339 form. + +The motivating use case is JSON Schema 2020-12 documents authored as TOML, but +`tomljson` is application-agnostic — anyone moving JSON-shaped data through TOML +can use it. ## Quick start @@ -33,31 +46,34 @@ assert_eq!(back, value); ## API surface -| Function | Purpose | -|---|---| -| `to_string(value)` | encode with default options | +| Function | Purpose | +| ----------------------------------------- | ------------------------------ | +| `to_string(value)` | encode with default options | | `to_string_with_options(value, &options)` | encode with custom placeholder | -| `from_str(s)` | decode with default options | -| `from_str_with_options(s, &options)` | decode with custom placeholder | +| `from_str(s)` | decode with default options | +| `from_str_with_options(s, &options)` | decode with custom placeholder | Plus: -- `TomlJsonOptions { null_placeholder: String, root_placeholder: String }` — configure both reserved strings if your data collides with the defaults. -- `DEFAULT_NULL_PLACEHOLDER` (`"__null__"`), `DEFAULT_ROOT_PLACEHOLDER` (`"__root__"`). + +- `TomlJsonOptions { null_placeholder: String, root_placeholder: String }` — + configure both reserved strings if your data collides with the defaults. +- `DEFAULT_NULL_PLACEHOLDER` (`"__null__"`), `DEFAULT_ROOT_PLACEHOLDER` + (`"__root__"`). - `Error`, `Result`. ## Encoding rules ### JSON → TOML (encode) -| Concern | Handling | -|---|---| -| `Json::Null` value (anywhere) | Substitute the placeholder string (default `"__null__"`) | -| Top-level non-table value (`bool`, scalar, array) | Wrap under the configured `root_placeholder` key (default `"__root__"`) | -| Top-level Object whose keys include `root_placeholder` | **Error** (`RootKeyCollision`) — encoder would emit a wrap-shaped TOML doc indistinguishable from a wrapped non-table; decoder would silently strip the user's data | -| `serde_json::Number` representable as `u64 > i64::MAX` | **Error** (`IntegerOutOfRange`) — TOML's signed 64-bit limit; JSON spec is wider | -| String value equal to the configured `null_placeholder` | **Error** (`PlaceholderCollision`) — the round-trip would be ambiguous | -| Strings shaped like TOML literals (`"42"`, `"true"`, `"2026-05-04"`, `"inf"`) | Always quoted on output; never coerced | -| Other JSON values (string, finite number, bool, array, object) | Direct emission via `toml_writer` primitives | +| Concern | Handling | +| ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Json::Null` value (anywhere) | Substitute the placeholder string (default `"__null__"`) | +| Top-level non-table value (`bool`, scalar, array) | Wrap under the configured `root_placeholder` key (default `"__root__"`) | +| Top-level Object whose keys include `root_placeholder` | **Error** (`RootKeyCollision`) — encoder would emit a wrap-shaped TOML doc indistinguishable from a wrapped non-table; decoder would silently strip the user's data | +| `serde_json::Number` representable as `u64 > i64::MAX` | **Error** (`IntegerOutOfRange`) — TOML's signed 64-bit limit; JSON spec is wider | +| String value equal to the configured `null_placeholder` | **Error** (`PlaceholderCollision`) — the round-trip would be ambiguous | +| Strings shaped like TOML literals (`"42"`, `"true"`, `"2026-05-04"`, `"inf"`) | Always quoted on output; never coerced | +| Other JSON values (string, finite number, bool, array, object) | Direct emission via `toml_writer` primitives | Internal structure on encode: @@ -67,25 +83,31 @@ Internal structure on encode: ### TOML → JSON (decode) -| Concern | Handling | -|---|---| -| TOML string equal to the configured placeholder | Decode as `Json::Null` | -| Root-level table containing exactly one key matching `root_placeholder` | Unwrap and return the inner value | +| Concern | Handling | +| ------------------------------------------------------------------------ | --------------------------------------------------- | +| TOML string equal to the configured placeholder | Decode as `Json::Null` | +| Root-level table containing exactly one key matching `root_placeholder` | Unwrap and return the inner value | | TOML datetime (any of `Date`, `Time`, `LocalDateTime`, `OffsetDateTime`) | Decode as `Json::String` in canonical RFC 3339 form | -| TOML float `+inf` / `-inf` / `NaN` | **Error** (`FloatNotRepresentable`) | -| TOML integer (`i64`) | Decode as `Json::Number` (i64-backed) | -| TOML finite float (`f64`) | Decode as `Json::Number` (f64-backed) | -| TOML bool, array, table | Recursively decode | -| Absent TOML key | Absent in JSON object — no special handling | -| TOML parse failure | **Error** (`Toml`) | +| TOML float `+inf` / `-inf` / `NaN` | **Error** (`FloatNotRepresentable`) | +| TOML integer (`i64`) | Decode as `Json::Number` (i64-backed) | +| TOML finite float (`f64`) | Decode as `Json::Number` (f64-backed) | +| TOML bool, array, table | Recursively decode | +| Absent TOML key | Absent in JSON object — no special handling | +| TOML parse failure | **Error** (`Toml`) | ## Design notes ### Why `+inf` / `-inf` / `NaN` error on decode -`serde_json::Number::from_f64` rejects non-finite floats. JSON itself has no syntax for them. JSON Schema's validation model can't compare against them either — `maximum: inf` is meaningless because *every* finite number satisfies it. +`serde_json::Number::from_f64` rejects non-finite floats. JSON itself has no +syntax for them. JSON Schema's validation model can't compare against them +either — `maximum: inf` is meaningless because _every_ finite number satisfies +it. -Producers wanting "no upper bound" should **omit** `maximum` rather than write `inf`. Storage layers (Parquet, Arrow, LanceDB) handle infinities natively, but the JSON validation layer cannot, and `tomljson` errors at the boundary so the limitation surfaces early instead of silently corrupting data. +Producers wanting "no upper bound" should **omit** `maximum` rather than write +`inf`. Storage layers (Parquet, Arrow, LanceDB) handle infinities natively, but +the JSON validation layer cannot, and `tomljson` errors at the boundary so the +limitation surfaces early instead of silently corrupting data. ### Why absent and null must be distinguished @@ -95,7 +117,9 @@ TOML's grammar has **no syntax for "key present, no value"**: x = # parse error ``` -But JSON treats `{}` and `{"x": null}` as different objects (key present, value null vs. key absent). To preserve this distinction across the round-trip, `tomljson` uses a placeholder string for null: +But JSON treats `{}` and `{"x": null}` as different objects (key present, value +null vs. key absent). To preserve this distinction across the round-trip, +`tomljson` uses a placeholder string for null: ```toml x = "__null__" # x is present, value is JSON null @@ -104,9 +128,15 @@ x = "__null__" # x is present, value is JSON null ## Limitations -- **No generic `Serialize`/`DeserializeOwned` wrapper.** Callers with typed structs convert at the boundary via `serde_json::to_value` / `serde_json::from_value`. -- **No streaming.** Both encode and decode operate on whole documents. The underlying `toml` crate has no streaming parser, and TOML's key-order freedom would defeat streaming on the decode side anyway. -- **Datetime decode is one-way.** Decoding produces JSON strings (RFC 3339); re-encoding those strings doesn't reconstruct TOML's native datetime form. JSON has no native datetime type. +- **No generic `Serialize`/`DeserializeOwned` wrapper.** Callers with typed + structs convert at the boundary via `serde_json::to_value` / + `serde_json::from_value`. +- **No streaming.** Both encode and decode operate on whole documents. The + underlying `toml` crate has no streaming parser, and TOML's key-order freedom + would defeat streaming on the decode side anyway. +- **Datetime decode is one-way.** Decoding produces JSON strings (RFC 3339); + re-encoding those strings doesn't reconstruct TOML's native datetime form. + JSON has no native datetime type. ## License diff --git a/docs/benchmarks/report.md b/docs/benchmarks/report.md index 35f50c2..8984254 100644 --- a/docs/benchmarks/report.md +++ b/docs/benchmarks/report.md @@ -1,50 +1,75 @@ # mdvs vs QMD — benchmark report -_Generated 2026-06-02 19:46_ -_Corpora: `example_kb`, `docs`_ -_mdvs 0.6.2 · QMD 2.5.2_ +_Generated 2026-06-02 19:46_ _Corpora: `example_kb`, `docs`_ _mdvs 0.6.2 · QMD +2.5.2_ -This report characterises how mdvs and QMD compare on warm/steady-state search latency, peak memory, build time, and output footprint. See [TODO-0166](../spec/todos/TODO-0166.md) for the framing and decisions behind what's measured (and what's deliberately not). +This report characterises how mdvs and QMD compare on warm/steady-state search +latency, peak memory, build time, and output footprint. See +[TODO-0166](../spec/todos/TODO-0166.md) for the framing and decisions behind +what's measured (and what's deliberately not). ## Methodology For each (tool, corpus, query) combination the runner records: -- **Warm/steady-state search latency** — `/usr/bin/time -l` wall time, median of N iterations after one warm-up invocation +- **Warm/steady-state search latency** — `/usr/bin/time -l` wall time, median of + N iterations after one warm-up invocation - **Peak resident set size** — maximum RSS observed during the query -- **CPU%** — derived from `(user + sys) / wall × 100`; indicates whether wall time was CPU-bound or I/O-bound -- **Index build time** — single timed run from clean state (`rm -rf .mdvs && mdvs build --force` for mdvs; `qmd collection add` + `qmd embed -f` for QMD) +- **CPU%** — derived from `(user + sys) / wall × 100`; indicates whether wall + time was CPU-bound or I/O-bound +- **Index build time** — single timed run from clean state + (`rm -rf .mdvs && mdvs build --force` for mdvs; `qmd collection add` + + `qmd embed -f` for QMD) - **Index size on disk** — `du -sk` after build -- **Output token count** — `tiktoken` cl100k_base over the result snippets, for `--limit 10` +- **Output token count** — `tiktoken` cl100k_base over the result snippets, for + `--limit 10` - **Tool footprint on disk** — binary + cached embedding/reranker models -Cold-start latency and page-fault counts are deliberately excluded; see [TODO-0166](../spec/todos/TODO-0166.md) for the rationale. +Cold-start latency and page-fault counts are deliberately excluded; see +[TODO-0166](../spec/todos/TODO-0166.md) for the rationale. -Each search runs three iterations preceded by one warm-up invocation, so the reported wall time and RSS reflect steady-state behaviour. +Each search runs three iterations preceded by one warm-up invocation, so the +reported wall time and RSS reflect steady-state behaviour. ## Reading the numbers fairly -The two tools have meaningfully different feature sets. Any conclusions drawn from these numbers should respect the following: - -- **QMD `query` runs LLM reranking and query expansion** on top of BM25 + vector. mdvs's hybrid mode uses RRF only — no LLM in the loop. The reranking step changes both latency and quality; comparing wall times alone understates QMD's quality work -- **QMD does AST-aware chunking** for source code (TypeScript, JavaScript, Python, Go, Rust); mdvs uses prose chunking via `text-splitter`'s `MarkdownSplitter`. On code-heavy corpora the chunking strategies will produce different recall/precision profiles, independent of search engine speed -- **mdvs has `--where` SQL filtering and frontmatter validation**; QMD has neither. These are feature presence, not performance, and don't appear in the metric tables -- **Embedding models differ in size and quality.** mdvs uses Model2Vec `potion-base-8M` (~30 MB static distillation); QMD uses `embeddinggemma-300M-Q8_0` (~300 MB GGUF). Smaller model → less memory and faster load, but a different quality ceiling -- **Default chunking and limits differ.** mdvs uses 1024-char chunks; QMD's default chunking produces roughly one chunk per file on this corpus. Result count and token count comparisons should be read with this in mind - -This benchmark measures **latency, footprint, and setup cost** under each tool's defaults. It does not measure ranking quality — that would require a labelled query set and is out of scope. +The two tools have meaningfully different feature sets. Any conclusions drawn +from these numbers should respect the following: + +- **QMD `query` runs LLM reranking and query expansion** on top of BM25 + + vector. mdvs's hybrid mode uses RRF only — no LLM in the loop. The reranking + step changes both latency and quality; comparing wall times alone understates + QMD's quality work +- **QMD does AST-aware chunking** for source code (TypeScript, JavaScript, + Python, Go, Rust); mdvs uses prose chunking via `text-splitter`'s + `MarkdownSplitter`. On code-heavy corpora the chunking strategies will produce + different recall/precision profiles, independent of search engine speed +- **mdvs has `--where` SQL filtering and frontmatter validation**; QMD has + neither. These are feature presence, not performance, and don't appear in the + metric tables +- **Embedding models differ in size and quality.** mdvs uses Model2Vec + `potion-base-8M` (~30 MB static distillation); QMD uses + `embeddinggemma-300M-Q8_0` (~300 MB GGUF). Smaller model → less memory and + faster load, but a different quality ceiling +- **Default chunking and limits differ.** mdvs uses 1024-char chunks; QMD's + default chunking produces roughly one chunk per file on this corpus. Result + count and token count comparisons should be read with this in mind + +This benchmark measures **latency, footprint, and setup cost** under each tool's +defaults. It does not measure ranking quality — that would require a labelled +query set and is out of scope. ## Test environment -| | | -|---|---| -| OS | `macOS-26.5-arm64-arm-64bit` | -| CPU arch | `arm64` | -| Python | `3.11.14` | -| mdvs version | `mdvs 0.6.2` | -| qmd version | `qmd 2.5.2` | -| Iterations per query | 3 (+ 1 warm-up) | -| --limit | 10 | +| | | +| -------------------- | ---------------------------- | +| OS | `macOS-26.5-arm64-arm-64bit` | +| CPU arch | `arm64` | +| Python | `3.11.14` | +| mdvs version | `mdvs 0.6.2` | +| qmd version | `qmd 2.5.2` | +| Iterations per query | 3 (+ 1 warm-up) | +| --limit | 10 | ## Corpus: `example_kb` (46 files) @@ -52,60 +77,70 @@ This benchmark measures **latency, footprint, and setup cost** under each tool's Both tools are set up fresh each run. The two phases are timed separately: -- **prepare** — mdvs `init` (schema inference) / QMD `collection add` (scan + chunk + metadata) -- **index** — mdvs `build --force` (scan + chunk + validate + embed) / QMD `embed -f` (vectors) +- **prepare** — mdvs `init` (schema inference) / QMD `collection add` (scan + + chunk + metadata) +- **index** — mdvs `build --force` (scan + chunk + validate + embed) / QMD + `embed -f` (vectors) -(mdvs bundles scan/chunk/validate into `build`; QMD splits them into `collection add`. The **total** is the comparable figure — raw files to a queryable index.) +(mdvs bundles scan/chunk/validate into `build`; QMD splits them into +`collection add`. The **total** is the comparable figure — raw files to a +queryable index.) -| | mdvs | QMD | -|---|---|---| -| Prepare (init / collection add) | 210 ms | 190 ms | -| Index (build / embed) | 280 ms | 2.81 s | -| **Total setup** | 490 ms | 3.00 s | -| Index peak RSS | 125 MB | 821 MB | -| Index on disk | 232.0 KB | 75.1 MB | -| Embedding/reranker models on disk | 59.0 MB | 2.10 GB | +| | mdvs | QMD | +| --------------------------------- | -------- | ------- | +| Prepare (init / collection add) | 210 ms | 190 ms | +| Index (build / embed) | 280 ms | 2.81 s | +| **Total setup** | 490 ms | 3.00 s | +| Index peak RSS | 125 MB | 821 MB | +| Index on disk | 232.0 KB | 75.1 MB | +| Embedding/reranker models on disk | 59.0 MB | 2.10 GB | ### Queries -| Kind | Query | mdvs mode | `--where` clause | -|---|---|---|---| -| `broad_semantic` | _"calibration baseline"_ | `semantic` | — | -| `narrow_semantic` | _"wavelet denoising replication"_ | `semantic` | — | -| `exact_phrase` | _"SPR-A1"_ | `fulltext` | — | -| `metadata_filtered` | _"calibration"_ | `hybrid` | `status = 'completed'` | -| `vague_multiword` | _"what went wrong with the spectrometer"_ | `hybrid` | — | +| Kind | Query | mdvs mode | `--where` clause | +| ------------------- | ----------------------------------------- | ---------- | ---------------------- | +| `broad_semantic` | _"calibration baseline"_ | `semantic` | — | +| `narrow_semantic` | _"wavelet denoising replication"_ | `semantic` | — | +| `exact_phrase` | _"SPR-A1"_ | `fulltext` | — | +| `metadata_filtered` | _"calibration"_ | `hybrid` | `status = 'completed'` | +| `vague_multiword` | _"what went wrong with the spectrometer"_ | `hybrid` | — | ### Search latency (warm, median of N) mdvs is reported in two configurations: -- **mdvs default** — runs as users typically invoke it; `auto_update` and `auto_build` in `mdvs.toml` cause a scan + frontmatter-validation + build-check pass before every search (~110 ms on this corpus) -- **mdvs engine-only** — same query with `--no-update --no-build`. Measures the search engine itself without the orchestration overhead. Closer to a like-for-like comparison with QMD, which has no equivalent feature - -| Kind | mdvs default | mdvs engine-only | mdvs RSS | mdvs CPU% | QMD mode | QMD wall | QMD RSS | QMD CPU% | -|---|---|---|---|---|---|---|---|---| -| `broad_semantic` | 240 ms | 240 ms | 123 MB | 29% | `vsearch` | 800 ms | 617 MB | 87% | -| `narrow_semantic` | 240 ms | 240 ms | 123 MB | 25% | `vsearch` | 800 ms | 624 MB | 87% | -| `exact_phrase` | 220 ms | 210 ms | 47.3 MB | 18% | `search` | 150 ms | 66.6 MB | 100% | -| `metadata_filtered` | 240 ms | 240 ms | 130 MB | 29% | — | — | — | — | -| `vague_multiword` | 240 ms | 240 ms | 127 MB | 29% | `query` | 700 ms | 638 MB | 106% | +- **mdvs default** — runs as users typically invoke it; `auto_update` and + `auto_build` in `mdvs.toml` cause a scan + frontmatter-validation + + build-check pass before every search (~110 ms on this corpus) +- **mdvs engine-only** — same query with `--no-update --no-build`. Measures the + search engine itself without the orchestration overhead. Closer to a + like-for-like comparison with QMD, which has no equivalent feature + +| Kind | mdvs default | mdvs engine-only | mdvs RSS | mdvs CPU% | QMD mode | QMD wall | QMD RSS | QMD CPU% | +| ------------------- | ------------ | ---------------- | -------- | --------- | --------- | -------- | ------- | -------- | +| `broad_semantic` | 240 ms | 240 ms | 123 MB | 29% | `vsearch` | 800 ms | 617 MB | 87% | +| `narrow_semantic` | 240 ms | 240 ms | 123 MB | 25% | `vsearch` | 800 ms | 624 MB | 87% | +| `exact_phrase` | 220 ms | 210 ms | 47.3 MB | 18% | `search` | 150 ms | 66.6 MB | 100% | +| `metadata_filtered` | 240 ms | 240 ms | 130 MB | 29% | — | — | — | — | +| `vague_multiword` | 240 ms | 240 ms | 127 MB | 29% | `query` | 700 ms | 638 MB | 106% | ### Output token count (snippets for `--limit 10`, `tiktoken` `cl100k_base`) -Token count matters when results are piped into a downstream LLM — fewer tokens = less context spent. +Token count matters when results are piped into a downstream LLM — fewer tokens += less context spent. -| Kind | mdvs result count | mdvs tokens | QMD result count | QMD tokens | -|---|---|---|---|---| -| `broad_semantic` | 10 | 1,712 | 10 | 444 | -| `narrow_semantic` | 10 | 1,373 | 8 | 467 | -| `exact_phrase` | 10 | 1,601 | 7 | 376 | -| `metadata_filtered` | 5 | 974 | — | — | -| `vague_multiword` | 10 | 1,518 | 10 | 607 | +| Kind | mdvs result count | mdvs tokens | QMD result count | QMD tokens | +| ------------------- | ----------------- | ----------- | ---------------- | ---------- | +| `broad_semantic` | 10 | 1,712 | 10 | 444 | +| `narrow_semantic` | 10 | 1,373 | 8 | 467 | +| `exact_phrase` | 10 | 1,601 | 7 | 376 | +| `metadata_filtered` | 5 | 974 | — | — | +| `vague_multiword` | 10 | 1,518 | 10 | 607 | ### Notes -- _qmd_: QMD uses a global ~/.cache/qmd/index.sqlite; index_size_bytes includes any unrelated user collections +- _qmd_: QMD uses a global ~/.cache/qmd/index.sqlite; index_size_bytes includes + any unrelated user collections - _qmd_: skipped 'metadata_filtered': qmd has no --where equivalent ## Corpus: `docs` (1669 files) @@ -114,58 +149,68 @@ Token count matters when results are piped into a downstream LLM — fewer token Both tools are set up fresh each run. The two phases are timed separately: -- **prepare** — mdvs `init` (schema inference) / QMD `collection add` (scan + chunk + metadata) -- **index** — mdvs `build --force` (scan + chunk + validate + embed) / QMD `embed -f` (vectors) +- **prepare** — mdvs `init` (schema inference) / QMD `collection add` (scan + + chunk + metadata) +- **index** — mdvs `build --force` (scan + chunk + validate + embed) / QMD + `embed -f` (vectors) -(mdvs bundles scan/chunk/validate into `build`; QMD splits them into `collection add`. The **total** is the comparable figure — raw files to a queryable index.) +(mdvs bundles scan/chunk/validate into `build`; QMD splits them into +`collection add`. The **total** is the comparable figure — raw files to a +queryable index.) -| | mdvs | QMD | -|---|---|---| -| Prepare (init / collection add) | 460 ms | 1.66 s | -| Index (build / embed) | 2.57 s | 743.11 s | -| **Total setup** | 3.03 s | 744.77 s | -| Index peak RSS | 355 MB | 1.08 GB | -| Index on disk | 31.7 MB | 75.1 MB | -| Embedding/reranker models on disk | 59.0 MB | 2.10 GB | +| | mdvs | QMD | +| --------------------------------- | ------- | -------- | +| Prepare (init / collection add) | 460 ms | 1.66 s | +| Index (build / embed) | 2.57 s | 743.11 s | +| **Total setup** | 3.03 s | 744.77 s | +| Index peak RSS | 355 MB | 1.08 GB | +| Index on disk | 31.7 MB | 75.1 MB | +| Embedding/reranker models on disk | 59.0 MB | 2.10 GB | ### Queries -| Kind | Query | mdvs mode | `--where` clause | -|---|---|---|---| -| `broad_semantic` | _"deploying applications to kubernetes"_ | `semantic` | — | -| `narrow_semantic` | _"rolling update strategy"_ | `semantic` | — | -| `exact_phrase` | _"kubectl apply"_ | `fulltext` | — | -| `metadata_filtered` | _"minikube"_ | `hybrid` | `content_type = 'tutorial'` | -| `vague_multiword` | _"how do I expose my service to the internet"_ | `hybrid` | — | +| Kind | Query | mdvs mode | `--where` clause | +| ------------------- | ---------------------------------------------- | ---------- | --------------------------- | +| `broad_semantic` | _"deploying applications to kubernetes"_ | `semantic` | — | +| `narrow_semantic` | _"rolling update strategy"_ | `semantic` | — | +| `exact_phrase` | _"kubectl apply"_ | `fulltext` | — | +| `metadata_filtered` | _"minikube"_ | `hybrid` | `content_type = 'tutorial'` | +| `vague_multiword` | _"how do I expose my service to the internet"_ | `hybrid` | — | ### Search latency (warm, median of N) mdvs is reported in two configurations: -- **mdvs default** — runs as users typically invoke it; `auto_update` and `auto_build` in `mdvs.toml` cause a scan + frontmatter-validation + build-check pass before every search (~110 ms on this corpus) -- **mdvs engine-only** — same query with `--no-update --no-build`. Measures the search engine itself without the orchestration overhead. Closer to a like-for-like comparison with QMD, which has no equivalent feature - -| Kind | mdvs default | mdvs engine-only | mdvs RSS | mdvs CPU% | QMD mode | QMD wall | QMD RSS | QMD CPU% | -|---|---|---|---|---|---|---|---|---| -| `broad_semantic` | 420 ms | 300 ms | 268 MB | 60% | `vsearch` | 800 ms | 630 MB | 91% | -| `narrow_semantic` | 450 ms | 290 ms | 267 MB | 60% | `vsearch` | 800 ms | 630 MB | 92% | -| `exact_phrase` | 430 ms | 280 ms | 267 MB | 60% | `search` | 160 ms | 72.6 MB | 94% | -| `metadata_filtered` | 420 ms | 310 ms | 267 MB | 59% | — | — | — | — | -| `vague_multiword` | 440 ms | 310 ms | 267 MB | 60% | `query` | 790 ms | 646 MB | 101% | +- **mdvs default** — runs as users typically invoke it; `auto_update` and + `auto_build` in `mdvs.toml` cause a scan + frontmatter-validation + + build-check pass before every search (~110 ms on this corpus) +- **mdvs engine-only** — same query with `--no-update --no-build`. Measures the + search engine itself without the orchestration overhead. Closer to a + like-for-like comparison with QMD, which has no equivalent feature + +| Kind | mdvs default | mdvs engine-only | mdvs RSS | mdvs CPU% | QMD mode | QMD wall | QMD RSS | QMD CPU% | +| ------------------- | ------------ | ---------------- | -------- | --------- | --------- | -------- | ------- | -------- | +| `broad_semantic` | 420 ms | 300 ms | 268 MB | 60% | `vsearch` | 800 ms | 630 MB | 91% | +| `narrow_semantic` | 450 ms | 290 ms | 267 MB | 60% | `vsearch` | 800 ms | 630 MB | 92% | +| `exact_phrase` | 430 ms | 280 ms | 267 MB | 60% | `search` | 160 ms | 72.6 MB | 94% | +| `metadata_filtered` | 420 ms | 310 ms | 267 MB | 59% | — | — | — | — | +| `vague_multiword` | 440 ms | 310 ms | 267 MB | 60% | `query` | 790 ms | 646 MB | 101% | ### Output token count (snippets for `--limit 10`, `tiktoken` `cl100k_base`) -Token count matters when results are piped into a downstream LLM — fewer tokens = less context spent. +Token count matters when results are piped into a downstream LLM — fewer tokens += less context spent. -| Kind | mdvs result count | mdvs tokens | QMD result count | QMD tokens | -|---|---|---|---|---| -| `broad_semantic` | 10 | 806 | 10 | 624 | -| `narrow_semantic` | 9 | 719 | 10 | 604 | -| `exact_phrase` | 10 | 869 | 10 | 440 | -| `metadata_filtered` | 10 | 1,193 | — | — | -| `vague_multiword` | 10 | 1,274 | 10 | 563 | +| Kind | mdvs result count | mdvs tokens | QMD result count | QMD tokens | +| ------------------- | ----------------- | ----------- | ---------------- | ---------- | +| `broad_semantic` | 10 | 806 | 10 | 624 | +| `narrow_semantic` | 9 | 719 | 10 | 604 | +| `exact_phrase` | 10 | 869 | 10 | 440 | +| `metadata_filtered` | 10 | 1,193 | — | — | +| `vague_multiword` | 10 | 1,274 | 10 | 563 | ### Notes -- _qmd_: QMD uses a global ~/.cache/qmd/index.sqlite; index_size_bytes includes any unrelated user collections +- _qmd_: QMD uses a global ~/.cache/qmd/index.sqlite; index_size_bytes includes + any unrelated user collections - _qmd_: skipped 'metadata_filtered': qmd has no --where equivalent diff --git a/docs/spec/architecture.md b/docs/spec/architecture.md index 42b67f5..a86aac0 100644 --- a/docs/spec/architecture.md +++ b/docs/spec/architecture.md @@ -1,17 +1,28 @@ # Architecture -Developer map of the mdvs codebase. For user-facing documentation see the [mdBook](../../book/src/SUMMARY.md). For CLI reference see `mdvs --help`. +Developer map of the mdvs codebase. For user-facing documentation see the +[mdBook](../../book/src/SUMMARY.md). For CLI reference see `mdvs --help`. ## Overview mdvs has two layers: -- **Validation layer** (init, update, check) — scans markdown, infers schema, validates frontmatter. No model needed. Operates on `mdvs.toml`. -- **Search layer** (build, search) — chunks markdown, embeds text, stores in a single Lance dataset, queries via LanceDB's native search (semantic / fulltext / hybrid). Requires an embedding model. Operates on `.mdvs/`. +- **Validation layer** (init, update, check) — scans markdown, infers schema, + validates frontmatter. No model needed. Operates on `mdvs.toml`. +- **Search layer** (build, search) — chunks markdown, embeds text, stores in a + single Lance dataset, queries via LanceDB's native search (semantic / fulltext + / hybrid). Requires an embedding model. Operates on `.mdvs/`. -A third install-time subsystem — **agent-harness scaffolding** (`mdvs scaffold {skill,snippet,hook}` install commands plus the `mdvs hook handle` runtime) — wires mdvs into Claude Code / Codex / Cursor / OpenCode / Antigravity from per-platform config files. Per-platform JSON shapes (envelope, install-time config) live as data, not Rust. See [scaffolding.md](scaffolding.md) for the full design. +A third install-time subsystem — **agent-harness scaffolding** +(`mdvs scaffold {skill,snippet,hook}` install commands plus the +`mdvs hook handle` runtime) — wires mdvs into Claude Code / Codex / Cursor / +OpenCode / Antigravity from per-platform config files. Per-platform JSON shapes +(envelope, install-time config) live as data, not Rust. See +[scaffolding.md](scaffolding.md) for the full design. -`mdvs.toml` is the single source of truth for schema. There is no lock file. Build metadata (model identity, chunk size, schema hash) is stored as Lance table-level key-value metadata. +`mdvs.toml` is the single source of truth for schema. There is no lock file. +Build metadata (model identity, chunk size, schema hash) is stored as Lance +table-level key-value metadata. ```mermaid graph LR @@ -52,17 +63,37 @@ graph LR Pipeline stages with the key type at each boundary: -1. **Scan** — walk directory, per-file frontmatter dispatch (YAML / TOML via `gray_matter`; JSON via `serde_json` directly) → `ScannedFiles` (`discover/scan.rs:46`) -2. **Type inference** — single pass, widen types across files → `FieldTypeInfo` map (`discover/infer/types.rs:12`, widening at `discover/field_type.rs:29`) -3. **Path inference** — build directory tree, collapse into glob patterns → `FieldPaths` (`discover/infer/paths.rs:12`) -4. **Constraint inference** — categorical heuristic on distinct values → `Option` (`discover/infer/constraints/mod.rs:13`) -5. **Preprocessor inference** — observed widening events drive `Vec` per field (`preprocess.rs::infer_value_stages`) -6. **Config generation** — combine inferred fields into TOML config → `MdvsToml` (`schema/config.rs`) -7. **Validation** — translate via `dsl_to_canonical`, compile per-field `jsonschema::Validator`, run Stage 2 preprocessors, validate, map errors → `Vec` (`cmd/check.rs`) -8. **Chunking** — semantic markdown splitting with line ranges → `Chunks` (`index/chunk.rs:20`) -9. **Embedding** — plain text → dense vector via model2vec → `Vec` (`index/embed.rs:34`) -10. **Storage** — write to the single `.mdvs/index.lance/` dataset via one of three paths in `cmd/build/write.rs::write_index_step`: skip (no delta + not full rebuild), full overwrite (`Backend::write_index` → `create_table(...).mode(Overwrite)`), or incremental (`Backend::write_index_incremental` → delete rows by file_id + append + refresh metadata + optimize). One row per chunk in either persist path. (`index/storage.rs`, `index/backend/`) -11. **Search** — `SearchMode`-dispatched LanceDB query (`nearest_to` / `full_text_search` / hybrid + RRF reranker) with `--where` translated to LanceDB's SQL filter; best-chunk-per-file dedupe in Rust → `Vec` (`index/backend.rs`) +1. **Scan** — walk directory, per-file frontmatter dispatch (YAML / TOML via + `gray_matter`; JSON via `serde_json` directly) → `ScannedFiles` + (`discover/scan.rs:46`) +2. **Type inference** — single pass, widen types across files → `FieldTypeInfo` + map (`discover/infer/types.rs:12`, widening at `discover/field_type.rs:29`) +3. **Path inference** — build directory tree, collapse into glob patterns → + `FieldPaths` (`discover/infer/paths.rs:12`) +4. **Constraint inference** — categorical heuristic on distinct values → + `Option` (`discover/infer/constraints/mod.rs:13`) +5. **Preprocessor inference** — observed widening events drive `Vec` + per field (`preprocess.rs::infer_value_stages`) +6. **Config generation** — combine inferred fields into TOML config → `MdvsToml` + (`schema/config.rs`) +7. **Validation** — translate via `dsl_to_canonical`, compile per-field + `jsonschema::Validator`, run Stage 2 preprocessors, validate, map errors → + `Vec` (`cmd/check.rs`) +8. **Chunking** — semantic markdown splitting with line ranges → `Chunks` + (`index/chunk.rs:20`) +9. **Embedding** — plain text → dense vector via model2vec → `Vec` + (`index/embed.rs:34`) +10. **Storage** — write to the single `.mdvs/index.lance/` dataset via one of + three paths in `cmd/build/write.rs::write_index_step`: skip (no delta + not + full rebuild), full overwrite (`Backend::write_index` → + `create_table(...).mode(Overwrite)`), or incremental + (`Backend::write_index_incremental` → delete rows by file_id + append + + refresh metadata + optimize). One row per chunk in either persist path. + (`index/storage.rs`, `index/backend/`) +11. **Search** — `SearchMode`-dispatched LanceDB query (`nearest_to` / + `full_text_search` / hybrid + RRF reranker) with `--where` translated to + LanceDB's SQL filter; best-chunk-per-file dedupe in Rust → `Vec` + (`index/backend.rs`) ## Module Tree @@ -151,138 +182,199 @@ src/ ### Discovery & Inference -| Type | Location | Role | -|------|----------|------| -| `ScannedFile` | `discover/scan.rs:32` | Parsed markdown: path, frontmatter as JSON, body, line offset | -| `ScannedFiles` | `discover/scan.rs:46` | Collection of scanned files, entry point via `::scan()` | -| `FieldType` | `discover/field_type.rs:8` | Recursive type enum (Boolean, Integer, Float, String, Date, DateTime, Array, Object). Date and DateTime auto-inferred from RFC 3339 strings (TODO-0007). | -| `FieldTypeInfo` | `discover/infer/types.rs:12` | Per-field widened type + file list + distinct values + occurrence count | -| `DirectoryTree` | `discover/infer/paths.rs:20` | Arena-based tree for glob pattern collapsing | -| `FieldPaths` | `discover/infer/paths.rs:12` | Inferred allowed + required glob patterns | -| `InferredField` | `discover/infer/mod.rs:26` | Complete field: type, paths, nullable, distinct values | -| `InferredSchema` | `discover/infer/mod.rs:71` | All inferred fields, sorted by name | +| Type | Location | Role | +| ---------------- | ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ScannedFile` | `discover/scan.rs:32` | Parsed markdown: path, frontmatter as JSON, body, line offset | +| `ScannedFiles` | `discover/scan.rs:46` | Collection of scanned files, entry point via `::scan()` | +| `FieldType` | `discover/field_type.rs:8` | Recursive type enum (Boolean, Integer, Float, String, Date, DateTime, Array, Object). Date and DateTime auto-inferred from RFC 3339 strings (TODO-0007). | +| `FieldTypeInfo` | `discover/infer/types.rs:12` | Per-field widened type + file list + distinct values + occurrence count | +| `DirectoryTree` | `discover/infer/paths.rs:20` | Arena-based tree for glob pattern collapsing | +| `FieldPaths` | `discover/infer/paths.rs:12` | Inferred allowed + required glob patterns | +| `InferredField` | `discover/infer/mod.rs:26` | Complete field: type, paths, nullable, distinct values | +| `InferredSchema` | `discover/infer/mod.rs:71` | All inferred fields, sorted by name | ### Configuration -| Type | Location | Role | -|------|----------|------| -| `MdvsToml` | `schema/config.rs` | Top-level config, single source of truth. `default_with_fields(fields, ignore)` synthesizes a minimal config from imported JSON Schema | -| `TomlField` | `schema/config.rs` | Per-field definition: type, allowed, required, nullable, constraints, **preprocess** | -| `FieldsConfig` | `schema/config.rs` | Fields section: ignore list, field definitions, inference thresholds | -| `FieldTypeSerde` | `schema/shared.rs` | TOML-serializable type enum (Scalar/Array/Object) | -| `ScanConfig` | `schema/shared.rs` | Glob pattern, include_bare_files, skip_gitignore | -| `EmbeddingModelConfig` | `schema/shared.rs` | Model identity: provider, name, revision | -| `ChunkingConfig` | `schema/shared.rs` | max_chunk_size | -| `Constraints` | `schema/constraints/mod.rs` | Serde layer: `categories`, `min`, `max`, `min_length`, `max_length`, `pattern` (all Option). `#[serde(deny_unknown_fields)]` | -| `ConstraintKind` | `schema/constraints/mod.rs` | Behavior layer enum: `Categories`, `Range { min, max }`, `Length { min, max }`, `Pattern(String)` | -| `ValueStage` | `preprocess.rs` | Stage 2 preprocessor enum: `CoerceToString`, `WidenIntToFloat`. Inherent methods: `applies_to`, `applicable_types`, `apply` (all exhaustive matches) | -| `Pipeline` | `preprocess.rs` | Composed preprocessor for one field. Built via `Pipeline::for_config`; runs via `apply_to_value` | +| Type | Location | Role | +| ---------------------- | --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `MdvsToml` | `schema/config.rs` | Top-level config, single source of truth. `default_with_fields(fields, ignore)` synthesizes a minimal config from imported JSON Schema | +| `TomlField` | `schema/config.rs` | Per-field definition: type, allowed, required, nullable, constraints, **preprocess** | +| `FieldsConfig` | `schema/config.rs` | Fields section: ignore list, field definitions, inference thresholds | +| `FieldTypeSerde` | `schema/shared.rs` | TOML-serializable type enum (Scalar/Array/Object) | +| `ScanConfig` | `schema/shared.rs` | Glob pattern, include_bare_files, skip_gitignore | +| `EmbeddingModelConfig` | `schema/shared.rs` | Model identity: provider, name, revision | +| `ChunkingConfig` | `schema/shared.rs` | max_chunk_size | +| `Constraints` | `schema/constraints/mod.rs` | Serde layer: `categories`, `min`, `max`, `min_length`, `max_length`, `pattern` (all Option). `#[serde(deny_unknown_fields)]` | +| `ConstraintKind` | `schema/constraints/mod.rs` | Behavior layer enum: `Categories`, `Range { min, max }`, `Length { min, max }`, `Pattern(String)` | +| `ValueStage` | `preprocess.rs` | Stage 2 preprocessor enum: `CoerceToString`, `WidenIntToFloat`. Inherent methods: `applies_to`, `applicable_types`, `apply` (all exhaustive matches) | +| `Pipeline` | `preprocess.rs` | Composed preprocessor for one field. Built via `Pipeline::for_config`; runs via `apply_to_value` | ### Index & Search -| Type | Location | Role | -|------|----------|------| -| `Chunk` | `index/chunk.rs:8` | Semantic chunk: index, start/end lines, plain text | -| `Chunks` | `index/chunk.rs:20` | Newtype wrapping `Vec`, created via `::new()` | -| `ModelConfig` | `index/embed.rs:9` | Resolved model config (enum: Model2Vec variant) | -| `Embedder` | `index/embed.rs:34` | Loaded model (enum: Model2Vec(StaticModel)) | -| `ChunkRow` | `index/storage.rs` | Per-chunk row built from a `Chunk` + its file's `FileRow` data + embedding | -| `FileRow` | `index/storage.rs` | Per-file frontmatter snapshot (filepath, data Struct, content_hash, built_at) — duplicated onto each of that file's chunk rows | -| `BuildMetadata` | `index/storage.rs` | Build config stored as Lance table-level kv metadata | -| `FileIndexEntry` | `index/storage.rs` | Lightweight projected read for incremental classification (file_id, filepath, content_hash) | -| `Backend` | `index/backend.rs` | Storage backend enum (only `Lance` variant) | -| `LanceBackend` | `index/backend/` | LanceDB connection + `write_index` (full rebuild) + `write_index_incremental` (delete + append + refresh + optimize) + `search` (mode-dispatched) + `--where` translator | -| `SearchMode` | `search.rs` | `Semantic` / `Fulltext` / `Hybrid` (default `Hybrid`) | -| `SearchHit` | `index/backend.rs` | Query result: filename, score, chunk lines, text | +| Type | Location | Role | +| ---------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `Chunk` | `index/chunk.rs:8` | Semantic chunk: index, start/end lines, plain text | +| `Chunks` | `index/chunk.rs:20` | Newtype wrapping `Vec`, created via `::new()` | +| `ModelConfig` | `index/embed.rs:9` | Resolved model config (enum: Model2Vec variant) | +| `Embedder` | `index/embed.rs:34` | Loaded model (enum: Model2Vec(StaticModel)) | +| `ChunkRow` | `index/storage.rs` | Per-chunk row built from a `Chunk` + its file's `FileRow` data + embedding | +| `FileRow` | `index/storage.rs` | Per-file frontmatter snapshot (filepath, data Struct, content_hash, built_at) — duplicated onto each of that file's chunk rows | +| `BuildMetadata` | `index/storage.rs` | Build config stored as Lance table-level kv metadata | +| `FileIndexEntry` | `index/storage.rs` | Lightweight projected read for incremental classification (file_id, filepath, content_hash) | +| `Backend` | `index/backend.rs` | Storage backend enum (only `Lance` variant) | +| `LanceBackend` | `index/backend/` | LanceDB connection + `write_index` (full rebuild) + `write_index_incremental` (delete + append + refresh + optimize) + `search` (mode-dispatched) + `--where` translator | +| `SearchMode` | `search.rs` | `Semantic` / `Fulltext` / `Hybrid` (default `Hybrid`) | +| `SearchHit` | `index/backend.rs` | Query result: filename, score, chunk lines, text | ### Output & Rendering -| Type | Location | Role | -|------|----------|------| -| `CommandResult` | `step.rs:90` | Command return: steps list + final result + elapsed_ms | -| `StepEntry` | `step.rs:34` | Completed / Failed / Skipped step | -| `Outcome` | `outcome/mod.rs:41` | Enum with one variant per step/command outcome | -| `Block` | `block.rs:11` | Rendering IR: Line, Table, Section | -| `Render` | `block.rs:56` | Trait: `render_compact()` / `render_verbose()` → `Vec` | -| `ViolationKind` | `output.rs:173` | Enum (derives `PartialOrd, Ord` — declaration order is the sort key): `MissingRequired`, `WrongType`, `Disallowed`, `NullNotAllowed`, `InvalidCategory`, `OutOfRange`, `FrontmatterUnrepresentable` | -| `FieldViolation` | `output.rs:197` | Grouped violation: field, kind, rule, files | -| `DiscoveredField` | `output.rs:64` | Inferred field for command output | -| `ChangedField` | `output.rs:88` | Field with detected changes (type, allowed, required, nullable) | -| `FieldChange` | `output.rs:98` | Enum of change kinds with old/new values | +| Type | Location | Role | +| ----------------- | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `CommandResult` | `step.rs:90` | Command return: steps list + final result + elapsed_ms | +| `StepEntry` | `step.rs:34` | Completed / Failed / Skipped step | +| `Outcome` | `outcome/mod.rs:41` | Enum with one variant per step/command outcome | +| `Block` | `block.rs:11` | Rendering IR: Line, Table, Section | +| `Render` | `block.rs:56` | Trait: `render_compact()` / `render_verbose()` → `Vec` | +| `ViolationKind` | `output.rs:173` | Enum (derives `PartialOrd, Ord` — declaration order is the sort key): `MissingRequired`, `WrongType`, `Disallowed`, `NullNotAllowed`, `InvalidCategory`, `OutOfRange`, `FrontmatterUnrepresentable` | +| `FieldViolation` | `output.rs:197` | Grouped violation: field, kind, rule, files | +| `DiscoveredField` | `output.rs:64` | Inferred field for command output | +| `ChangedField` | `output.rs:88` | Field with detected changes (type, allowed, required, nullable) | +| `FieldChange` | `output.rs:98` | Enum of change kinds with old/new values | ## Enum Dispatch Pattern -mdvs uses enum-based dispatch instead of trait objects for all runtime polymorphism. Key enums: +mdvs uses enum-based dispatch instead of trait objects for all runtime +polymorphism. Key enums: - `FieldType` — type system (6 variants) - `Backend` — storage backend (1 variant: `Lance`) -- `Embedder` / `ModelConfig` — embedding provider (1 variant: Model2Vec; Ollama planned) -- `ConstraintKind` — constraint behavior (4 variants: Categories, Range, Length, Pattern) -- `ValueStage` — Stage 2 preprocessors (2 variants: CoerceToString, WidenIntToFloat; more planned) -- `SearchMode` — search dispatch (3 variants: Semantic, Fulltext, Hybrid; default Hybrid) +- `Embedder` / `ModelConfig` — embedding provider (1 variant: Model2Vec; Ollama + planned) +- `ConstraintKind` — constraint behavior (4 variants: Categories, Range, Length, + Pattern) +- `ValueStage` — Stage 2 preprocessors (2 variants: CoerceToString, + WidenIntToFloat; more planned) +- `SearchMode` — search dispatch (3 variants: Semantic, Fulltext, Hybrid; + default Hybrid) - `Outcome` — step/command results (~20 variants) - `Block` — rendering IR (3 variants) -Rationale: single binary (no feature flags for variant selection), exhaustive match guarantees compile-time coverage, no dynamic dispatch overhead. Adding a new variant = add to the enum + implement match arms. +Rationale: single binary (no feature flags for variant selection), exhaustive +match guarantees compile-time coverage, no dynamic dispatch overhead. Adding a +new variant = add to the enum + implement match arms. ## Validation Pipeline -Post-Wave-B, runtime validation goes through the `jsonschema` crate. Hand-rolled per-value validators have been deleted. +Post-Wave-B, runtime validation goes through the `jsonschema` crate. Hand-rolled +per-value validators have been deleted. ### Stages -1. **Translation** — `dsl_to_canonical(config)` in `schema/json_schema.rs` translates `MdvsToml` into a JSON Schema 2020-12 document. Strict types: `FieldType::String` emits `{"type": "string"}` (no permissive set). `FieldType::Date` and `FieldType::DateTime` emit `{"type": "string", "format": "date"}` and `{"type": "string", "format": "date-time"}`. Path-scoping is carried as `x-mdvs.allowed` / `x-mdvs.required` per property; preprocessor stages as `x-mdvs.preprocess`. -2. **Gate** — `validate_mdvs_schema(schema)` checks an allow-list of keywords + a hard-reject list for unsupported JSON Schema features (`oneOf`, `$ref`, etc.) with explanatory messages. The `format` keyword is allowed only with values in `ALLOWED_FORMATS = ["date", "date-time"]`; other format strings are rejected with a "use pattern" hint. Run on any schema before it enters the pipeline (`init --from-jsonschema`, `check --jsonschema`). +1. **Translation** — `dsl_to_canonical(config)` in `schema/json_schema.rs` + translates `MdvsToml` into a JSON Schema 2020-12 document. Strict types: + `FieldType::String` emits `{"type": "string"}` (no permissive set). + `FieldType::Date` and `FieldType::DateTime` emit + `{"type": "string", "format": "date"}` and + `{"type": "string", "format": "date-time"}`. Path-scoping is carried as + `x-mdvs.allowed` / `x-mdvs.required` per property; preprocessor stages as + `x-mdvs.preprocess`. +2. **Gate** — `validate_mdvs_schema(schema)` checks an allow-list of keywords + + a hard-reject list for unsupported JSON Schema features (`oneOf`, `$ref`, + etc.) with explanatory messages. The `format` keyword is allowed only with + values in `ALLOWED_FORMATS = ["date", "date-time"]`; other format strings are + rejected with a "use pattern" hint. Run on any schema before it enters the + pipeline (`init --from-jsonschema`, `check --jsonschema`). 3. **Compile** — `validate()` in `cmd/check/validate.rs` builds per-call: - - `FieldValidators::build` — one `jsonschema::Validator` per field via `jsonschema::options().should_validate_formats(true).build()`. Format validation is enabled so `date` and `date-time` are checked at runtime, not just annotated. - - `build_field_metas(config)` (in `cmd/check/field_meta.rs`) — per-field `FieldMeta` with compiled `GlobSet`s for `allowed` / `required` (so the inner loop doesn't recompile globs per file) and a cached `FieldType::try_from`. - - `Pipeline::for_config` — Stage 2 preprocessors per field, applied before validation so values arrive normalized. -4. **Validate** — `check_field_values` walks each frontmatter leaf. For each `(field, file)` it: runs `preprocess::strict_subtype_check` (Rust-side), then the Stage 2 pipeline, then `validator.is_valid()` as a fast path — only when that returns false does it iterate `validator.iter_errors()` and map each error via `map_validation_error` (exhaustive match over `ValidationErrorKind`) into an mdvs `ViolationKind`. The path-scoping `Disallowed` check uses the precomputed `FieldMeta::allowed` `GlobSet`. `check_required_fields` similarly uses `FieldMeta::required`. `collect_violations` then sorts the accumulator by `(field, kind, rule)` outer / `path` inner — `Vec` output is byte-stable across runs. + - `FieldValidators::build` — one `jsonschema::Validator` per field via + `jsonschema::options().should_validate_formats(true).build()`. Format + validation is enabled so `date` and `date-time` are checked at runtime, not + just annotated. + - `build_field_metas(config)` (in `cmd/check/field_meta.rs`) — per-field + `FieldMeta` with compiled `GlobSet`s for `allowed` / `required` (so the + inner loop doesn't recompile globs per file) and a cached + `FieldType::try_from`. + - `Pipeline::for_config` — Stage 2 preprocessors per field, applied before + validation so values arrive normalized. +4. **Validate** — `check_field_values` walks each frontmatter leaf. For each + `(field, file)` it: runs `preprocess::strict_subtype_check` (Rust-side), then + the Stage 2 pipeline, then `validator.is_valid()` as a fast path — only when + that returns false does it iterate `validator.iter_errors()` and map each + error via `map_validation_error` (exhaustive match over + `ValidationErrorKind`) into an mdvs `ViolationKind`. The path-scoping + `Disallowed` check uses the precomputed `FieldMeta::allowed` `GlobSet`. + `check_required_fields` similarly uses `FieldMeta::required`. + `collect_violations` then sorts the accumulator by `(field, kind, rule)` + outer / `path` inner — `Vec` output is byte-stable across + runs. ### Error mapping (exhaustive) -| `ValidationErrorKind` (jsonschema) | `ViolationKind` (mdvs) | -|---|---| -| `Type` (instance is null) | `NullNotAllowed` | -| `Type` (other) | `WrongType` | -| `Enum`, `Constant` | `InvalidCategory` | -| `Minimum`, `Maximum`, `ExclusiveMinimum`, `ExclusiveMaximum`, `MultipleOf` | `OutOfRange` | -| `MinLength`, `MaxLength`, `MinItems`, `MaxItems`, `UniqueItems` | `OutOfRange` | -| `Pattern` | `WrongType` | -| `Format` | `WrongType` (rule = `format `) | -| `Required` | `MissingRequired` | -| `AdditionalProperties` | `Disallowed` | - -Gate-rejected variants (`OneOfNotValid`, `Referencing`, etc.) bucket to a "schema gate should reject this — please report" path; reaching one means the gate has drifted. +| `ValidationErrorKind` (jsonschema) | `ViolationKind` (mdvs) | +| -------------------------------------------------------------------------- | ------------------------------------ | +| `Type` (instance is null) | `NullNotAllowed` | +| `Type` (other) | `WrongType` | +| `Enum`, `Constant` | `InvalidCategory` | +| `Minimum`, `Maximum`, `ExclusiveMinimum`, `ExclusiveMaximum`, `MultipleOf` | `OutOfRange` | +| `MinLength`, `MaxLength`, `MinItems`, `MaxItems`, `UniqueItems` | `OutOfRange` | +| `Pattern` | `WrongType` | +| `Format` | `WrongType` (rule = `format `) | +| `Required` | `MissingRequired` | +| `AdditionalProperties` | `Disallowed` | + +Gate-rejected variants (`OneOfNotValid`, `Referencing`, etc.) bucket to a +"schema gate should reject this — please report" path; reaching one means the +gate has drifted. ### Round-trip -`canonical_to_dsl` reverses the translation for `mdvs init --from-jsonschema`. It accepts only the exact shapes `dsl_to_canonical` produces; arbitrary hand-written schemas that pass the gate but use unusual structures (e.g. enum without `type`) error out with a clear message. `mdvs export-jsonschema` round-trips losslessly back to the same `[[fields.field]]` definitions including constraints, path-scoping, and `preprocess` arrays. +`canonical_to_dsl` reverses the translation for `mdvs init --from-jsonschema`. +It accepts only the exact shapes `dsl_to_canonical` produces; arbitrary +hand-written schemas that pass the gate but use unusual structures (e.g. enum +without `type`) error out with a clear message. `mdvs export-jsonschema` +round-trips losslessly back to the same `[[fields.field]]` definitions including +constraints, path-scoping, and `preprocess` arrays. ## Constraint Architecture -`Constraints` (`schema/constraints/mod.rs`) carries the serde layer: flat `Option<…>` fields mapping directly to `[fields.field.constraints]` in TOML. Fields: `categories`, `min`, `max`, `min_length`, `max_length`, `pattern`. `#[serde(deny_unknown_fields)]` rejects typos. `categories` is closed-set — mutually exclusive with everything else. +`Constraints` (`schema/constraints/mod.rs`) carries the serde layer: flat +`Option<…>` fields mapping directly to `[fields.field.constraints]` in TOML. +Fields: `categories`, `min`, `max`, `min_length`, `max_length`, `pattern`. +`#[serde(deny_unknown_fields)]` rejects typos. `categories` is closed-set — +mutually exclusive with everything else. -`ConstraintKind` (`schema/constraints/mod.rs`) is the behavior enum: `Categories`, `Range { min, max }`, `Length { min, max }`, `Pattern(String)`. Each kind has a `schema/constraints/.rs` submodule with one config-time function: +`ConstraintKind` (`schema/constraints/mod.rs`) is the behavior enum: +`Categories`, `Range { min, max }`, `Length { min, max }`, `Pattern(String)`. +Each kind has a `schema/constraints/.rs` submodule with one config-time +function: -- `validate_for_type(field_name, field_type)` → `Option` (is this kind applicable to this field type?) +- `validate_for_type(field_name, field_type)` → `Option` (is this kind + applicable to this field type?) -There is no `validate_value()` at this layer anymore — value-time checks are emitted as JSON Schema keywords by `dsl_to_canonical` and run by `jsonschema`. +There is no `validate_value()` at this layer anymore — value-time checks are +emitted as JSON Schema keywords by `dsl_to_canonical` and run by `jsonschema`. -`Constraints::validate_config()` runs config-time checks: per-kind type applicability, plus the categories-is-mutually-exclusive rule. Pairwise constraints (e.g. `min` ≤ `max`) are sanity-checked here too. +`Constraints::validate_config()` runs config-time checks: per-kind type +applicability, plus the categories-is-mutually-exclusive rule. Pairwise +constraints (e.g. `min` ≤ `max`) are sanity-checked here too. -Inference logic for constraints lives in `discover/infer/constraints/`. Adding a new constraint kind means: serde field on `Constraints`, variant on `ConstraintKind`, submodule under `schema/constraints/`, JSON Schema emission in `dsl_to_canonical`, optional inference in `discover/infer/constraints/`. +Inference logic for constraints lives in `discover/infer/constraints/`. Adding a +new constraint kind means: serde field on `Constraints`, variant on +`ConstraintKind`, submodule under `schema/constraints/`, JSON Schema emission in +`dsl_to_canonical`, optional inference in `discover/infer/constraints/`. ## Preprocessor Pipeline -`preprocess.rs` introduces three preprocessing stages by position. Only Stage 2 has built-ins in v0; Stage 1 (`FieldNameStage`) and Stage 3 (`DocumentStage`) are empty enums maintained as scaffolding. +`preprocess.rs` introduces three preprocessing stages by position. Only Stage 2 +has built-ins in v0; Stage 1 (`FieldNameStage`) and Stage 3 (`DocumentStage`) +are empty enums maintained as scaffolding. -**Stage 2 (`ValueStage`)** is a per-field, per-value transform applied **before** jsonschema validation. Built-ins: +**Stage 2 (`ValueStage`)** is a per-field, per-value transform applied +**before** jsonschema validation. Built-ins: -| Variant | Applies to | Behavior | -|---|---|---| -| `CoerceToString` | `String`, `Array(String)` | Non-string JSON value → its `to_string()` representation | -| `WidenIntToFloat` | `Float`, `Array(Float)` | Integer → equivalent float | +| Variant | Applies to | Behavior | +| ----------------- | ------------------------- | -------------------------------------------------------- | +| `CoerceToString` | `String`, `Array(String)` | Non-string JSON value → its `to_string()` representation | +| `WidenIntToFloat` | `Float`, `Array(Float)` | Integer → equivalent float | Each variant has three inherent methods, all exhaustive matches: @@ -292,56 +384,96 @@ Each variant has three inherent methods, all exhaustive matches: Adding a new variant fails compilation in three places. -**Inference auto-populates `preprocess`** from observed type-widening events. `FieldTypeInfo.observed_types: Vec` collects raw observation types per field. `infer_value_stages(observed, final_type)` returns the implied stages: +**Inference auto-populates `preprocess`** from observed type-widening events. +`FieldTypeInfo.observed_types: Vec` collects raw observation types +per field. `infer_value_stages(observed, final_type)` returns the implied +stages: -- `CoerceToString` when the final type is `String` and observations include non-string types. -- `WidenIntToFloat` when the final type is `Float` and observations include `Integer`. +- `CoerceToString` when the final type is `String` and observations include + non-string types. +- `WidenIntToFloat` when the final type is `Float` and observations include + `Integer`. No implicit defaults — `preprocess = []` means strict. -**`MdvsToml::validate()` invariant 5** (post-Wave-B) enforces that each preprocess entry is applicable to its field type, and that the list contains no duplicates. +**`MdvsToml::validate()` invariant 5** (post-Wave-B) enforces that each +preprocess entry is applicable to its field type, and that the list contains no +duplicates. ### Strict subtype prechecks -For some preprocessor stages, **the absence of the stage** must enforce a check that JSON Schema can't express. These checks run in Rust, before the preprocessor pipeline and before `jsonschema::Validator`. +For some preprocessor stages, **the absence of the stage** must enforce a check +that JSON Schema can't express. These checks run in Rust, before the +preprocessor pipeline and before `jsonschema::Validator`. -`CoerceToString`'s absence is enforced naturally: `{"type": "string"}` rejects non-strings. +`CoerceToString`'s absence is enforced naturally: `{"type": "string"}` rejects +non-strings. -`WidenIntToFloat`'s absence requires a Rust-side check because JSON Schema can't distinguish `Value::Number(5)` (i64-backed) from `Value::Number(5.0)` (f64-backed) — both match `"number"`, both match `"integer"` (per JSON Schema 2020-12, "integer" matches any number with zero fractional part). YAML and TOML preserve the int/float distinction at parse time (serde_json does too), but JSON Schema operates on the value's mathematical content, not its serde representation. +`WidenIntToFloat`'s absence requires a Rust-side check because JSON Schema can't +distinguish `Value::Number(5)` (i64-backed) from `Value::Number(5.0)` +(f64-backed) — both match `"number"`, both match `"integer"` (per JSON Schema +2020-12, "integer" matches any number with zero fractional part). YAML and TOML +preserve the int/float distinction at parse time (serde_json does too), but JSON +Schema operates on the value's mathematical content, not its serde +representation. -`preprocess::strict_subtype_check(field, field_type, value) -> Option` is called from `cmd/check.rs::check_field_values` before `pipeline.apply_to_value`. When it returns `Some(detail)`: +`preprocess::strict_subtype_check(field, field_type, value) -> Option` +is called from `cmd/check.rs::check_field_values` before +`pipeline.apply_to_value`. When it returns `Some(detail)`: -- A `ViolationKind::WrongType` violation is emitted with rule `format!("type {}", field.field_type)` and the returned detail (e.g. `"got Integer"`, `"got Integer at index 1"`). -- The preprocessor pipeline and the jsonschema validator are skipped for that value — no double violation. +- A `ViolationKind::WrongType` violation is emitted with rule + `format!("type {}", field.field_type)` and the returned detail (e.g. + `"got Integer"`, `"got Integer at index 1"`). +- The preprocessor pipeline and the jsonschema validator are skipped for that + value — no double violation. -Current scope: `Float` and `Array(Float)` fields without `WidenIntToFloat` in `preprocess` reject integer-backed values. Future ValueStages with a similar "absence-must-be-enforced-in-Rust" requirement extend the same function. +Current scope: `Float` and `Array(Float)` fields without `WidenIntToFloat` in +`preprocess` reject integer-backed values. Future ValueStages with a similar +"absence-must-be-enforced-in-Rust" requirement extend the same function. ## Date and DateTime types (TODO-0007) -Two type variants for time-shaped strings. Both store the canonical wire form per RFC 3339: +Two type variants for time-shaped strings. Both store the canonical wire form +per RFC 3339: -| Type | Wire format | JSON Schema | Arrow type | Validation | -|---|---|---|---|---| -| `Date` | `YYYY-MM-DD` | `{"type": "string", "format": "date"}` | `Date32` (days since 1970-01-01) | jsonschema `format: date` + chrono parse | -| `DateTime` | `YYYY-MM-DDTHH:MM:SS[.frac]` | `{"type": "string", "format": "date-time"}` | `Timestamp(Millisecond, "UTC")` | jsonschema `format: date-time` + `chrono::DateTime::parse_from_rfc3339` | +| Type | Wire format | JSON Schema | Arrow type | Validation | +| ---------- | --------------------------------------- | ------------------------------------------- | -------------------------------- | ----------------------------------------------------------------------- | +| `Date` | `YYYY-MM-DD` | `{"type": "string", "format": "date"}` | `Date32` (days since 1970-01-01) | jsonschema `format: date` + chrono parse | +| `DateTime` | `YYYY-MM-DDTHH:MM:SS[.frac]` | `{"type": "string", "format": "date-time"}` | `Timestamp(Millisecond, "UTC")` | jsonschema `format: date-time` + `chrono::DateTime::parse_from_rfc3339` | ### Inference -`From<&Value> for FieldType` checks `Value::String(s)` against two regex+chrono gates: -- `looks_like_date_time(s)` — RFC 3339 datetime regex + `chrono::DateTime::parse_from_rfc3339(s).is_ok()`. Checked first. -- `looks_like_date(s)` — RFC 3339 full-date regex + `chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").is_ok()`. +`From<&Value> for FieldType` checks `Value::String(s)` against two regex+chrono +gates: + +- `looks_like_date_time(s)` — RFC 3339 datetime regex + + `chrono::DateTime::parse_from_rfc3339(s).is_ok()`. Checked first. +- `looks_like_date(s)` — RFC 3339 full-date regex + + `chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").is_ok()`. -Shapes are disjoint (datetime requires `T`, date forbids it). Per-value detection; widening picks up mixed-observation cases via the standard rules: `Date + Date → Date`, `DateTime + DateTime → DateTime`, `Date + DateTime → String` (cross-shape), `Date + non-string → String`. Strict — one non-matching observation downgrades the whole field to String. +Shapes are disjoint (datetime requires `T`, date forbids it). Per-value +detection; widening picks up mixed-observation cases via the standard rules: +`Date + Date → Date`, `DateTime + DateTime → DateTime`, +`Date + DateTime → String` (cross-shape), `Date + non-string → String`. Strict — +one non-matching observation downgrades the whole field to String. ### Storage normalization -DateTime values are normalized to UTC at storage time (`dt.with_timezone(&chrono::Utc).timestamp_millis()`). `2024-04-02T16:14:30+02:00` and `2024-04-02T14:14:30Z` are the same absolute moment and store identically. The original offset is intentionally discarded; the Arrow column carries `Some("UTC")` as its timezone metadata. +DateTime values are normalized to UTC at storage time +(`dt.with_timezone(&chrono::Utc).timestamp_millis()`). +`2024-04-02T16:14:30+02:00` and `2024-04-02T14:14:30Z` are the same absolute +moment and store identically. The original offset is intentionally discarded; +the Arrow column carries `Some("UTC")` as its timezone metadata. -Date and DateTime do **not** support any preprocessor — there's no `parse-loose-date` opt-in. A string either parses as RFC 3339 or it falls back to String. +Date and DateTime do **not** support any preprocessor — there's no +`parse-loose-date` opt-in. A string either parses as RFC 3339 or it falls back +to String. ## Dotted-name leaf flattening (Wave C / TODO-0097) -mdvs.toml is **flat**: every `[[fields.field]]` declares one leaf, named by a dotted path. A nested frontmatter shape — shown here as YAML, but the same in TOML tables or JSON objects — like: +mdvs.toml is **flat**: every `[[fields.field]]` declares one leaf, named by a +dotted path. A nested frontmatter shape — shown here as YAML, but the same in +TOML tables or JSON objects — like: ```yaml calibration: @@ -364,61 +496,108 @@ type = "Float" **Three layers, three shapes, one truth:** -| Layer | Shape | Why | -|---|---|---| -| `mdvs.toml` | Flat list of dotted-name leaves | Per-leaf nullability, per-leaf path-scoping, readable in plain TOML | -| Canonical JSON Schema (`dsl_to_canonical`) | Nested `properties` tree | Standard JSON Schema; what `jsonschema::Validator` consumes | +| Layer | Shape | Why | +| -------------------------------------------------- | ---------------------------------------------- | -------------------------------------------------------------------------------------------- | +| `mdvs.toml` | Flat list of dotted-name leaves | Per-leaf nullability, per-leaf path-scoping, readable in plain TOML | +| Canonical JSON Schema (`dsl_to_canonical`) | Nested `properties` tree | Standard JSON Schema; what `jsonschema::Validator` consumes | | Arrow `data` Struct column (built in `storage.rs`) | Nested Struct mirroring the source frontmatter | LanceDB's SQL dot-access (`WHERE data.calibration.baseline.wavelength > 800`) works natively | -| Source frontmatter (YAML / TOML / JSON) | Naturally nested | Unchanged | +| Source frontmatter (YAML / TOML / JSON) | Naturally nested | Unchanged | -The translator (`dsl_to_canonical`) reconstructs the nested shape from dotted names; `canonical_to_dsl` reverses it. Storage transposes the flat list back into a synthetic `FieldType::Object` tree before building Arrow arrays. Validation navigates frontmatter via dotted paths (`navigate_dotted` in `cmd/check.rs`). +The translator (`dsl_to_canonical`) reconstructs the nested shape from dotted +names; `canonical_to_dsl` reverses it. Storage transposes the flat list back +into a synthetic `FieldType::Object` tree before building Arrow arrays. +Validation navigates frontmatter via dotted paths (`navigate_dotted` in +`cmd/check.rs`). ### Array-of-Object rejection (TODO-0155) -Top-level Object is rejected (invariant 6), and `Array(Object{...})` is rejected at three layers: +Top-level Object is rejected (invariant 6), and `Array(Object{...})` is rejected +at three layers: -1. **Parser** (`FieldTypeSerde::parse`) — `type = "Array(Object{...})"` fails to deserialize. -2. **Inference** (`InferredSchema::infer`) — observed Array-of-mapping fields are partitioned out of `fields` into `dropped`, and a stderr warning fires when init/update/build/check consumes the schema. -3. **Config-load** (`MdvsToml::validate` invariant 9) — defense in depth for `--from-jsonschema` imports and programmatic construction that bypass the parser. +1. **Parser** (`FieldTypeSerde::parse`) — `type = "Array(Object{...})"` fails to + deserialize. +2. **Inference** (`InferredSchema::infer`) — observed Array-of-mapping fields + are partitioned out of `fields` into `dropped`, and a stderr warning fires + when init/update/build/check consumes the schema. +3. **Config-load** (`MdvsToml::validate` invariant 9) — defense in depth for + `--from-jsonschema` imports and programmatic construction that bypass the + parser. -`FieldType::Object` remains in the Rust enum because Wave C's storage layer synthesizes it transiently from dotted-name leaves (`transpose_to_storage_type` builds `Object` trees before Arrow encoding). It cannot reach the serde surface. A first-class on-disk representation for Array-of-structured-item is tracked in TODO-0156; the v0 workaround is parallel scalar arrays. +`FieldType::Object` remains in the Rust enum because Wave C's storage layer +synthesizes it transiently from dotted-name leaves (`transpose_to_storage_type` +builds `Object` trees before Arrow encoding). It cannot reach the serde surface. +A first-class on-disk representation for Array-of-structured-item is tracked in +TODO-0156; the v0 workaround is parallel scalar arrays. ### Shape conflicts -Invariant 8 rejects configs that declare a name as both a leaf and a parent of nested leaves (e.g., `meta` *and* `meta.author`). This catches structural ambiguity at config-load time, before the translator runs. +Invariant 8 rejects configs that declare a name as both a leaf and a parent of +nested leaves (e.g., `meta` _and_ `meta.author`). This catches structural +ambiguity at config-load time, before the translator runs. ### Literal-dot frontmatter keys -A frontmatter key with a literal dot — `"foo.bar": "..."` in YAML, `"foo.bar" = "..."` in TOML, `"foo.bar": "..."` in JSON — conflicts with mdvs's dotted-name convention and is rejected at scan time via `FrontmatterUnrepresentable`. Users with such keys must restructure their frontmatter (or wait for a future Stage 1 field-name preprocessor that could remap them). +A frontmatter key with a literal dot — `"foo.bar": "..."` in YAML, +`"foo.bar" = "..."` in TOML, `"foo.bar": "..."` in JSON — conflicts with mdvs's +dotted-name convention and is rejected at scan time via +`FrontmatterUnrepresentable`. Users with such keys must restructure their +frontmatter (or wait for a future Stage 1 field-name preprocessor that could +remap them). ## Output Pipeline Three-stage rendering: 1. **Data** — command produces an `Outcome` struct (in `outcome/commands/`) -2. **Blocks** — `Render` trait (`block.rs:56`) converts outcome to `Vec` via `render_compact()` or `render_verbose()` -3. **Format** — `CommandResult::render(format, verbose)` (`step.rs`) dispatches to `format_pretty()`, `format_markdown()` (both in `render.rs`), or `serde_json::to_string_pretty()` based on the requested `OutputFormat`. - -`CommandResult` (`step.rs`) holds `Vec` (pipeline steps) + `Result` (final result). Verbose mode renders steps + result; compact renders result only. JSON uses `#[serde(untagged)]` on `Outcome` for flat serialization. - -Every command in `main.rs` follows the same dispatch pattern: call `run()`, check `has_failed()`, resolve `OutputFormat` via `resolve_output_format()` (CLI flag > `mdvs.toml`'s `default_output_format` > hard default `pretty`), call `result.render()`, print, set exit code. +2. **Blocks** — `Render` trait (`block.rs:56`) converts outcome to `Vec` + via `render_compact()` or `render_verbose()` +3. **Format** — `CommandResult::render(format, verbose)` (`step.rs`) dispatches + to `format_pretty()`, `format_markdown()` (both in `render.rs`), or + `serde_json::to_string_pretty()` based on the requested `OutputFormat`. + +`CommandResult` (`step.rs`) holds `Vec` (pipeline steps) + +`Result` (final result). Verbose mode renders steps + +result; compact renders result only. JSON uses `#[serde(untagged)]` on `Outcome` +for flat serialization. + +Every command in `main.rs` follows the same dispatch pattern: call `run()`, +check `has_failed()`, resolve `OutputFormat` via `resolve_output_format()` (CLI +flag > `mdvs.toml`'s `default_output_format` > hard default `pretty`), call +`result.render()`, print, set exit code. ## Incremental Build Build uses content hashing to avoid re-embedding unchanged files (`cmd/build/`): -1. **Hash** — `content_hash()` in `index/storage.rs` uses xxh3 on the markdown body (after frontmatter extraction). -2. **Classify** — `classify::classify_files` compares scanned files against `FileIndexEntry` projected from the existing Lance dataset and returns a `ClassifyData` with `needs_embedding`, `retained_chunks`, `removed_file_ids`, and the file_id map: +1. **Hash** — `content_hash()` in `index/storage.rs` uses xxh3 on the markdown + body (after frontmatter extraction). +2. **Classify** — `classify::classify_files` compares scanned files against + `FileIndexEntry` projected from the existing Lance dataset and returns a + `ClassifyData` with `needs_embedding`, `retained_chunks`, `removed_file_ids`, + and the file_id map: - **New** — no previous entry - **Edited** — hash differs - **Unchanged** — hash matches - **Removed** — in index but not in scan -3. **Skip model** — if no files need embedding, model loading is skipped entirely. +3. **Skip model** — if no files need embedding, model loading is skipped + entirely. 4. **Three-way write** — `write::write_index_step` dispatches the persist step: - - **Skip** when not a full rebuild AND `removed_count == 0` AND `new_chunks_count == 0`. The skip predicate uses chunk count (not file count) because empty-body files like Hugo `_index.md` are always classified as needing embedding but produce zero chunks. Returns `WriteOutcome::Skipped`, recorded as a `StepEntry::Skipped` (silent in text output, `"status": "skipped"` in JSON). - - **Full overwrite** when `full_rebuild` is true (first build or `--force`) — `Backend::write_index` calls `create_table(...).mode(Overwrite)` and rebuilds the FTS + (above 10k chunks) IVF-PQ indexes inside the new table. - - **Incremental** otherwise — `Backend::write_index_incremental` deletes the rows for `file_ids_to_clear` (new + edited + removed file_ids), appends the freshly embedded chunk slice, refreshes the schema metadata via `NativeTable::replace_schema_metadata`, and runs `optimize(All)` so the existing FTS + vector indexes incorporate the delta without a full rebuild. -5. **Force** — `--force` triggers full rebuild (path 2 above) regardless of the delta. Config changes (model, chunk size, prefix) also require `--force`. + - **Skip** when not a full rebuild AND `removed_count == 0` AND + `new_chunks_count == 0`. The skip predicate uses chunk count (not file + count) because empty-body files like Hugo `_index.md` are always classified + as needing embedding but produce zero chunks. Returns + `WriteOutcome::Skipped`, recorded as a `StepEntry::Skipped` (silent in text + output, `"status": "skipped"` in JSON). + - **Full overwrite** when `full_rebuild` is true (first build or `--force`) — + `Backend::write_index` calls `create_table(...).mode(Overwrite)` and + rebuilds the FTS + (above 10k chunks) IVF-PQ indexes inside the new table. + - **Incremental** otherwise — `Backend::write_index_incremental` deletes the + rows for `file_ids_to_clear` (new + edited + removed file_ids), appends the + freshly embedded chunk slice, refreshes the schema metadata via + `NativeTable::replace_schema_metadata`, and runs `optimize(All)` so the + existing FTS + vector indexes incorporate the delta without a full rebuild. +5. **Force** — `--force` triggers full rebuild (path 2 above) regardless of the + delta. Config changes (model, chunk size, prefix) also require `--force`. ## Storage Layout @@ -427,63 +606,97 @@ Build uses content hashing to avoid re-embedding unchanged files (`cmd/build/`): index.lance/ — Lance dataset, one row per chunk, plus FTS + (optionally) vector indexes ``` -**Chunk row columns**: `chunk_id` (Utf8), `file_id` (Utf8), `chunk_index` (Int32), `start_line` (Int32), `end_line` (Int32), `chunk_text` (Utf8 — persisted so FTS can index it and verbose snippets read it directly), `embedding` (FixedSizeList), `filepath` (Utf8), `content_hash` (Utf8), `data` (Struct — children are frontmatter fields), `built_at` (Timestamp). Column constants at `index/storage.rs`. +**Chunk row columns**: `chunk_id` (Utf8), `file_id` (Utf8), `chunk_index` +(Int32), `start_line` (Int32), `end_line` (Int32), `chunk_text` (Utf8 — +persisted so FTS can index it and verbose snippets read it directly), +`embedding` (FixedSizeList), `filepath` (Utf8), `content_hash` +(Utf8), `data` (Struct — children are frontmatter fields), `built_at` +(Timestamp). Column constants at `index/storage.rs`. -**Indexes inside the dataset**: a BM25 inverted index on `chunk_text` is created at every build. A cosine IVF-PQ vector index on `embedding` is created only above `VECTOR_INDEX_MIN_ROWS = 10_000`; smaller vaults rely on LanceDB's exact flat scan. +**Indexes inside the dataset**: a BM25 inverted index on `chunk_text` is created +at every build. A cosine IVF-PQ vector index on `embedding` is created only +above `VECTOR_INDEX_MIN_ROWS = 10_000`; smaller vaults rely on LanceDB's exact +flat scan. -**Build metadata** — stored as Lance table-level kv metadata. Keys prefixed `mdvs.`. Composed of `EmbeddingModelConfig` + `ChunkingConfig` + `internal_prefix` + `schema_hash` + `built_at` + `glob`. Comparisons via `PartialEq` on `BuildMetadata` (`index/storage.rs`). +**Build metadata** — stored as Lance table-level kv metadata. Keys prefixed +`mdvs.`. Composed of `EmbeddingModelConfig` + `ChunkingConfig` + +`internal_prefix` + `schema_hash` + `built_at` + `glob`. Comparisons via +`PartialEq` on `BuildMetadata` (`index/storage.rs`). -**Internal prefix** — kept for forward compatibility but currently a no-op since LanceDB column names match the constants directly. Configurable in `[search]`. Changing prefix requires `--force` rebuild. +**Internal prefix** — kept for forward compatibility but currently a no-op since +LanceDB column names match the constants directly. Configurable in `[search]`. +Changing prefix requires `--force` rebuild. ## Search Execution -Search dispatches on `SearchMode` (`search.rs`) and delegates to LanceDB (`index/backend.rs`): +Search dispatches on `SearchMode` (`search.rs`) and delegates to LanceDB +(`index/backend.rs`): -1. **Translate `--where`** — the clause is rewritten so bare frontmatter field names get a `data.` prefix (`translate_where_to_struct`); scalar SQL function calls are left as-is; `Array(Float)` field references early-error (TODO-0159). +1. **Translate `--where`** — the clause is rewritten so bare frontmatter field + names get a `data.` prefix (`translate_where_to_struct`); scalar SQL function + calls are left as-is; `Array(Float)` field references early-error + (TODO-0159). 2. **Build the query** — `lancedb::table.query()` plus: - **Semantic** — `.nearest_to(query_embedding).distance_type(Cosine)` - **Fulltext** — `.full_text_search(FullTextSearchQuery::new(query))` - **Hybrid** — both of the above + `.rerank(RrfReranker::default())` - - All modes apply `.only_if()` if a filter was given and `.limit(limit × OVER_FETCH_FACTOR)` where `OVER_FETCH_FACTOR = 3`. -3. **Consume the stream** — collect chunk rows with their per-mode score column (`_distance` → `1 - d`, `_score`, or `_relevance_score`). -4. **Best-chunk-per-file dedupe** — runs in Rust after the stream completes; the over-fetch factor compensates for chunks lost to dedupe. Final list is trimmed to `--limit`. -5. **Verbose snippet** — read directly from the persisted `chunk_text` column on the winning row (no second file read). + - All modes apply `.only_if()` if a filter was given and + `.limit(limit × OVER_FETCH_FACTOR)` where `OVER_FETCH_FACTOR = 3`. +3. **Consume the stream** — collect chunk rows with their per-mode score column + (`_distance` → `1 - d`, `_score`, or `_relevance_score`). +4. **Best-chunk-per-file dedupe** — runs in Rust after the stream completes; the + over-fetch factor compensates for chunks lost to dedupe. Final list is + trimmed to `--limit`. +5. **Verbose snippet** — read directly from the persisted `chunk_text` column on + the winning row (no second file read). ## Config Validation `MdvsToml::validate()` at `schema/config.rs` checks nine invariants: -1. **Mutual exclusion** — a field cannot appear in both `[fields].ignore` and `[[fields.field]]` -2. **Valid glob format** — all globs in `allowed`/`required` must end with `/*` or `/**` (or be `*` / `**`) -3. **Required covered by allowed** — every `required` glob must be covered by some `allowed` glob -4. **Constraint validity** — if a field has `[fields.field.constraints]`, the constraint must be valid for the field's type (type applicability + well-formed values + pairwise compatibility) -5. **Preprocess applicability** — each `preprocess` entry must be applicable to the field's type, and the list must contain no duplicates -6. **No top-level Object** — `[[fields.field]]` cannot use `Object` as its top-level type (per TODO-0097 / Wave C); nested data is expressed via dotted-name leaf fields. -7. **Dotted name well-formedness** — field names must not start or end with `.`, nor contain empty segments (`..`). Names without dots are unaffected. -8. **No shape conflicts** — a name cannot be declared both as a leaf and as a parent of nested leaves (e.g., `meta` *and* `meta.author`). -9. **No Array(Object)** — `Array(Object{...})` is not representable on disk (TODO-0155); rejected anywhere in any field's type tree. Defense in depth alongside the parser rejection. +1. **Mutual exclusion** — a field cannot appear in both `[fields].ignore` and + `[[fields.field]]` +2. **Valid glob format** — all globs in `allowed`/`required` must end with `/*` + or `/**` (or be `*` / `**`) +3. **Required covered by allowed** — every `required` glob must be covered by + some `allowed` glob +4. **Constraint validity** — if a field has `[fields.field.constraints]`, the + constraint must be valid for the field's type (type applicability + + well-formed values + pairwise compatibility) +5. **Preprocess applicability** — each `preprocess` entry must be applicable to + the field's type, and the list must contain no duplicates +6. **No top-level Object** — `[[fields.field]]` cannot use `Object` as its + top-level type (per TODO-0097 / Wave C); nested data is expressed via + dotted-name leaf fields. +7. **Dotted name well-formedness** — field names must not start or end with `.`, + nor contain empty segments (`..`). Names without dots are unaffected. +8. **No shape conflicts** — a name cannot be declared both as a leaf and as a + parent of nested leaves (e.g., `meta` _and_ `meta.author`). +9. **No Array(Object)** — `Array(Object{...})` is not representable on disk + (TODO-0155); rejected anywhere in any field's type tree. Defense in depth + alongside the parser rejection. All invariants bail on first error via `anyhow::bail!()`. ## External Dependencies -| Crate | Purpose | -|-------|---------| -| `clap` | CLI parsing (derive mode) | -| `lancedb` + `lance-index` | Storage + native vector / FTS / hybrid search | -| `arrow` | In-memory columnar format (batches handed to LanceDB) | -| `futures` | Stream consumption from LanceDB query results | -| `gray_matter` | YAML + TOML frontmatter extraction (JSON parsed natively via `serde_json::Deserializer::byte_offset`) | -| `jsonschema` | JSON Schema 2020-12 per-value validator (Wave B engine) | -| `tomljson` | Workspace crate: lossless TOML↔JSON for schema loading (`.toml` JSON Schemas) | -| `text-splitter` | Semantic markdown chunking | -| `pulldown-cmark` | Markdown to plain text | -| `model2vec-rs` | CPU-only static embedding (POTION models) | -| `indextree` | Arena-based tree for directory inference | -| `ignore` + `globset` | File walking with .gitignore/.mdvsignore | -| `tabled` | Box-drawing table rendering | -| `tracing` + `tracing-tree` | Structured stderr logging | -| `xxhash-rust` | Content hashing (xxh3) for incremental build | -| `tokio` | Async runtime (required by LanceDB) | -| `serde` + `toml` + `serde_json` | Serialization (TOML config, JSON frontmatter) | -| `anyhow` | Error handling | +| Crate | Purpose | +| ------------------------------- | ----------------------------------------------------------------------------------------------------- | +| `clap` | CLI parsing (derive mode) | +| `lancedb` + `lance-index` | Storage + native vector / FTS / hybrid search | +| `arrow` | In-memory columnar format (batches handed to LanceDB) | +| `futures` | Stream consumption from LanceDB query results | +| `gray_matter` | YAML + TOML frontmatter extraction (JSON parsed natively via `serde_json::Deserializer::byte_offset`) | +| `jsonschema` | JSON Schema 2020-12 per-value validator (Wave B engine) | +| `tomljson` | Workspace crate: lossless TOML↔JSON for schema loading (`.toml` JSON Schemas) | +| `text-splitter` | Semantic markdown chunking | +| `pulldown-cmark` | Markdown to plain text | +| `model2vec-rs` | CPU-only static embedding (POTION models) | +| `indextree` | Arena-based tree for directory inference | +| `ignore` + `globset` | File walking with .gitignore/.mdvsignore | +| `tabled` | Box-drawing table rendering | +| `tracing` + `tracing-tree` | Structured stderr logging | +| `xxhash-rust` | Content hashing (xxh3) for incremental build | +| `tokio` | Async runtime (required by LanceDB) | +| `serde` + `toml` + `serde_json` | Serialization (TOML config, JSON frontmatter) | +| `anyhow` | Error handling | diff --git a/docs/spec/archive/01-terminology.md b/docs/spec/archive/01-terminology.md index 3c53f7e..96b6615 100644 --- a/docs/spec/archive/01-terminology.md +++ b/docs/spec/archive/01-terminology.md @@ -2,7 +2,8 @@ **Status: DRAFT** -Canonical definitions for all terms used across mdvs specs. Every spec references this document rather than redefining terms. +Canonical definitions for all terms used across mdvs specs. Every spec +references this document rather than redefining terms. --- @@ -10,35 +11,55 @@ Canonical definitions for all terms used across mdvs specs. Every spec reference ### Vault -The target directory of markdown files that mdvs indexes. Any directory containing `.md` files — an Obsidian vault, a Hugo content directory, a Zettelkasten folder, a flat notes directory. mdvs makes no assumptions about the directory's structure or tooling. +The target directory of markdown files that mdvs indexes. Any directory +containing `.md` files — an Obsidian vault, a Hugo content directory, a +Zettelkasten folder, a flat notes directory. mdvs makes no assumptions about the +directory's structure or tooling. ### Frontmatter -YAML, TOML, or JSON metadata block at the top of a markdown file, delimited by `---` (YAML) or `+++` (TOML). Extracted by `gray_matter`. Not all files have frontmatter; files without it are still indexed with NULL metadata. +YAML, TOML, or JSON metadata block at the top of a markdown file, delimited by +`---` (YAML) or `+++` (TOML). Extracted by `gray_matter`. Not all files have +frontmatter; files without it are still indexed with NULL metadata. ### Field -A single key in a file's frontmatter (e.g., `title`, `tags`, `date`). Fields have inferred or explicitly declared types. Defined in `mfv.toml` / `mdvs.toml`. +A single key in a file's frontmatter (e.g., `title`, `tags`, `date`). Fields +have inferred or explicitly declared types. Defined in `mfv.toml` / `mdvs.toml`. ### Field Schema -The set of all field definitions in `mfv.toml` / `mdvs.toml`, including types and validation rules (`allowed`/`required` glob patterns, `pattern`, `values`). Shared between `mfv` and `mdvs`. +The set of all field definitions in `mfv.toml` / `mdvs.toml`, including types +and validation rules (`allowed`/`required` glob patterns, `pattern`, `values`). +Shared between `mfv` and `mdvs`. ### Chunk -A segment of a markdown file's body, produced by semantic splitting via `text-splitter`'s `MarkdownSplitter`. Each chunk is a self-contained piece of content that respects a configurable maximum size. Chunks are the unit of embedding and vector search — search results resolve to chunks, which map back to files via `filename`. +A segment of a markdown file's body, produced by semantic splitting via +`text-splitter`'s `MarkdownSplitter`. Each chunk is a self-contained piece of +content that respects a configurable maximum size. Chunks are the unit of +embedding and vector search — search results resolve to chunks, which map back +to files via `filename`. ### Plain Text -The result of stripping all markdown syntax from a chunk via `pulldown-cmark`. Only `Event::Text(...)` content is retained. This clean text is what gets embedded and stored in the `plain_text` column. The stripping removes bold, italic, links, code fences, headings markers, list markers, etc. +The result of stripping all markdown syntax from a chunk via `pulldown-cmark`. +Only `Event::Text(...)` content is retained. This clean text is what gets +embedded and stored in the `plain_text` column. The stripping removes bold, +italic, links, code fences, headings markers, list markers, etc. ### Embedding -A fixed-size floating-point vector (`Vec`) representing the semantic meaning of a chunk's plain text. Produced by a static embedding model. Stored as `FixedSizeList` in the chunks Parquet file where the list size equals the model's output dimension (e.g., 256). +A fixed-size floating-point vector (`Vec`) representing the semantic +meaning of a chunk's plain text. Produced by a static embedding model. Stored as +`FixedSizeList` in the chunks Parquet file where the list size equals +the model's output dimension (e.g., 256). ### Content Hash -A hash (xxhash or blake3) of the full file content (frontmatter + body). Stored per-file in `mdvs.lock` `[[file]]` entries. Used for incremental builds — only files whose hash has changed are reprocessed. +A hash (xxhash or blake3) of the full file content (frontmatter + body). Stored +per-file in `mdvs.lock` `[[file]]` entries. Used for incremental builds — only +files whose hash has changed are reprocessed. --- @@ -46,35 +67,51 @@ A hash (xxhash or blake3) of the full file content (frontmatter + body). Stored ### Static Embedding Model -A model that embeds text by tokenizing, looking up pre-computed token vectors in a matrix, and mean-pooling. No transformer forward pass, no GPU required, no context window. Inference is O(tokens) lookups. mdvs supports two formats — Model2Vec and Sentence Transformers StaticEmbedding — via a universal loader. See [Workflow: Model Loading](30-workflows/model-loading.md). +A model that embeds text by tokenizing, looking up pre-computed token vectors in +a matrix, and mean-pooling. No transformer forward pass, no GPU required, no +context window. Inference is O(tokens) lookups. mdvs supports two formats — +Model2Vec and Sentence Transformers StaticEmbedding — via a universal loader. +See [Workflow: Model Loading](30-workflows/model-loading.md). ### Model2Vec -The static embedding format used by MinishLab's POTION models. Files: `embeddings.safetensors` (tensor key `"embeddings"`), `tokenizer.json`, `config.json`. +The static embedding format used by MinishLab's POTION models. Files: +`embeddings.safetensors` (tensor key `"embeddings"`), `tokenizer.json`, +`config.json`. ### Sentence Transformers StaticEmbedding -The static embedding format used by Sentence Transformers. Files: `model.safetensors` (tensor key `"embedding.weight"`), `tokenizer.json`, plus ST pipeline configs (ignored by mdvs). Some ST models support Matryoshka truncation. +The static embedding format used by Sentence Transformers. Files: +`model.safetensors` (tensor key `"embedding.weight"`), `tokenizer.json`, plus ST +pipeline configs (ignored by mdvs). Some ST models support Matryoshka +truncation. ### POTION Model -A specific family of Model2Vec models from `minishlab`. Available in various sizes: `potion-base-2M` (64-dim), `potion-base-32M`, `potion-retrieval-32M` (512-dim), `potion-multilingual-128M`. +A specific family of Model2Vec models from `minishlab`. Available in various +sizes: `potion-base-2M` (64-dim), `potion-base-32M`, `potion-retrieval-32M` +(512-dim), `potion-multilingual-128M`. ### Matryoshka Truncation -A training technique (Matryoshka Representation Learning) where embeddings can be truncated to smaller dimensions with minimal quality loss. For example, a 1024-dim model truncated to 256-dim. Supported by some ST StaticEmbedding models. Configured via `truncate_dim` in `mdvs.toml`. +A training technique (Matryoshka Representation Learning) where embeddings can +be truncated to smaller dimensions with minimal quality loss. For example, a +1024-dim model truncated to 256-dim. Supported by some ST StaticEmbedding +models. Configured via `truncate_dim` in `mdvs.toml`. ### Model Identity -Three values that uniquely identify the model used to produce embeddings in an artifact: +Three values that uniquely identify the model used to produce embeddings in an +artifact: -| Field | Source | Purpose | -|---|---|---| -| **Model ID** | HuggingFace repo ID (e.g., `minishlab/potion-multilingual-128M`) | Identifies the model family | -| **Model Dimension** | Output vector size (e.g., 256) | Schema validation for embedding column | -| **Model Revision** | Git commit SHA of the downloaded snapshot | Detects silent model weight updates | +| Field | Source | Purpose | +| ------------------- | ---------------------------------------------------------------- | -------------------------------------- | +| **Model ID** | HuggingFace repo ID (e.g., `minishlab/potion-multilingual-128M`) | Identifies the model family | +| **Model Dimension** | Output vector size (e.g., 256) | Schema validation for embedding column | +| **Model Revision** | Git commit SHA of the downloaded snapshot | Detects silent model weight updates | -Stored in `mdvs.lock` `[build]` section. See [Model Mismatch Workflow](30-workflows/model-mismatch.md). +Stored in `mdvs.lock` `[build]` section. See +[Model Mismatch Workflow](30-workflows/model-mismatch.md). --- @@ -82,23 +119,33 @@ Stored in `mdvs.lock` `[build]` section. See [Model Mismatch Workflow](30-workfl ### Artifact -The `.mdvs/` directory at the root of the vault, containing compressed Parquet files produced by `mdvs build`. The searchable index. Analogous to `target/` in Cargo. Should be `.gitignore`-d. +The `.mdvs/` directory at the root of the vault, containing compressed Parquet +files produced by `mdvs build`. The searchable index. Analogous to `target/` in +Cargo. Should be `.gitignore`-d. ### `files.parquet` -One row per markdown file. Contains the filename (primary key), dynamic field columns from the schema, a JSON metadata column for fields not in the schema, and a content hash. See [Storage Schema](20-storage/schema.md). +One row per markdown file. Contains the filename (primary key), dynamic field +columns from the schema, a JSON metadata column for fields not in the schema, +and a content hash. See [Storage Schema](20-storage/schema.md). ### `chunks.parquet` -One row per semantic chunk. Contains chunk ID, parent filename, chunk index, nearest heading, plain text, embedding vector, and character count. +One row per semantic chunk. Contains chunk ID, parent filename, chunk index, +nearest heading, plain text, embedding vector, and character count. ### DataFusion -Pure Rust SQL query engine operating on Apache Arrow columnar data. Replaces DuckDB. Registers Parquet files as tables and executes SQL queries (JOIN, GROUP BY, WHERE) over them. Used for metadata filtering and note-level ranking during search. +Pure Rust SQL query engine operating on Apache Arrow columnar data. Replaces +DuckDB. Registers Parquet files as tables and executes SQL queries (JOIN, GROUP +BY, WHERE) over them. Used for metadata filtering and note-level ranking during +search. ### Parquet -Apache columnar file format for persistence. Supports compression (snappy/zstd), efficient column reads, and schema evolution. Used as the storage format in `.mdvs/`. +Apache columnar file format for persistence. Supports compression (snappy/zstd), +efficient column reads, and schema evolution. Used as the storage format in +`.mdvs/`. --- @@ -106,11 +153,20 @@ Apache columnar file format for persistence. Supports compression (snappy/zstd), ### `mfv.toml` / `mdvs.toml` -Field schema file shared between `mfv` and `mdvs`. Defines field types and validation rules (`allowed`/`required` glob patterns). Generated by `mfv init` or `mdvs init`. `mdvs.toml` additionally contains search-specific sections (`[model]`, `[chunking]`, `[behavior]`, `[search]`). See [Configuration](40-configuration/frontmatter-toml.md) and [Configuration: mdvs.toml](40-configuration/mdvs-toml.md). +Field schema file shared between `mfv` and `mdvs`. Defines field types and +validation rules (`allowed`/`required` glob patterns). Generated by `mfv init` +or `mdvs init`. `mdvs.toml` additionally contains search-specific sections +(`[model]`, `[chunking]`, `[behavior]`, `[search]`). See +[Configuration](40-configuration/frontmatter-toml.md) and +[Configuration: mdvs.toml](40-configuration/mdvs-toml.md). ### Lock File (`mfv.lock` / `mdvs.lock`) -Auto-generated snapshot of the resolved state (like `Cargo.lock`). `mfv.lock` contains field observations (which fields exist in which files). `mdvs.lock` is a superset: same field observations plus `[[file]]` entries with content hashes for staleness detection and a `[build]` section with artifact metadata (model identity, timestamps). +Auto-generated snapshot of the resolved state (like `Cargo.lock`). `mfv.lock` +contains field observations (which fields exist in which files). `mdvs.lock` is +a superset: same field observations plus `[[file]]` entries with content hashes +for staleness detection and a `[build]` section with artifact metadata (model +identity, timestamps). --- @@ -118,15 +174,22 @@ Auto-generated snapshot of the resolved state (like `Cargo.lock`). `mfv.lock` co ### mdvs -The full semantic search CLI binary. Superset of mfv — does everything mfv does plus frontmatter content querying and vector search. Depends on `mdvs-schema` and `mfv` crates. Commands: `init`, `build`, `search`, `update`, `check`, `clean`, `info`. +The full semantic search CLI binary. Superset of mfv — does everything mfv does +plus frontmatter content querying and vector search. Depends on `mdvs-schema` +and `mfv` crates. Commands: `init`, `build`, `search`, `update`, `check`, +`clean`, `info`. ### mfv (Markdown Frontmatter Validator) -Standalone frontmatter validation CLI binary (~2MB). No embeddings, no search. Independently publishable. Useful for CI pipelines, blog linting, documentation validation. Commands: `init`, `update`, `check`. +Standalone frontmatter validation CLI binary (~2MB). No embeddings, no search. +Independently publishable. Useful for CI pipelines, blog linting, documentation +validation. Commands: `init`, `update`, `check`. ### `mdvs-schema` -Shared library crate. Defines field types, the type system, TOML parsing, field discovery, tree inference, and lock file types. Dependency of both `mfv` and `mdvs`. +Shared library crate. Defines field types, the type system, TOML parsing, field +discovery, tree inference, and lock file types. Dependency of both `mfv` and +`mdvs`. --- @@ -134,19 +197,29 @@ Shared library crate. Defines field types, the type system, TOML parsing, field ### Build -The process of creating or updating the `.mdvs/` artifact from markdown files. Scans files, extracts frontmatter, chunks content, computes embeddings, writes Parquet files. Incremental by default (only reprocesses changed files). `mdvs build --full` for clean rebuild. Analogous to `cargo build`. +The process of creating or updating the `.mdvs/` artifact from markdown files. +Scans files, extracts frontmatter, chunks content, computes embeddings, writes +Parquet files. Incremental by default (only reprocesses changed files). +`mdvs build --full` for clean rebuild. Analogous to `cargo build`. ### Incremental Build -The default build mode. Compares content hashes in `mdvs.lock` against the filesystem to determine which files are new, modified, deleted, or unchanged. Only reprocesses changed files. +The default build mode. Compares content hashes in `mdvs.lock` against the +filesystem to determine which files are new, modified, deleted, or unchanged. +Only reprocesses changed files. ### Staleness -Whether the artifact (`.mdvs/`) is up to date with the files on disk. Configured via `on_stale` in `mdvs.toml`: `"auto"` (default) transparently builds before search, `"strict"` errors if stale. CLI overrides: `--build` / `--no-build` on `mdvs search`. +Whether the artifact (`.mdvs/`) is up to date with the files on disk. Configured +via `on_stale` in `mdvs.toml`: `"auto"` (default) transparently builds before +search, `"strict"` errors if stale. CLI overrides: `--build` / `--no-build` on +`mdvs search`. ### Note-Level Ranking -Search ranking strategy that groups chunk-level results by file. A file's score is the **maximum similarity** (minimum cosine distance) across all its chunks. The snippet and heading shown are from the best-matching chunk. +Search ranking strategy that groups chunk-level results by file. A file's score +is the **maximum similarity** (minimum cosine distance) across all its chunks. +The snippet and heading shown are from the best-matching chunk. --- diff --git a/docs/spec/archive/10-crates/mdvs-schema/spec.md b/docs/spec/archive/10-crates/mdvs-schema/spec.md index 93febe8..1af64f6 100644 --- a/docs/spec/archive/10-crates/mdvs-schema/spec.md +++ b/docs/spec/archive/10-crates/mdvs-schema/spec.md @@ -2,18 +2,22 @@ **Status: DRAFT** -**Cross-references:** [Terminology](../../01-terminology.md) | [Configuration](../../40-configuration/frontmatter-toml.md) +**Cross-references:** [Terminology](../../01-terminology.md) | +[Configuration](../../40-configuration/frontmatter-toml.md) --- ## Overview -Shared library crate containing field definitions, the type system, TOML parsing, field discovery, tree inference, and lock file types. Dependency of both `mfv` and `mdvs`. Has no knowledge of DataFusion, embeddings, or search. +Shared library crate containing field definitions, the type system, TOML +parsing, field discovery, tree inference, and lock file types. Dependency of +both `mfv` and `mdvs`. Has no knowledge of DataFusion, embeddings, or search. **Responsibilities:** - Parse `mfv.toml` / `mdvs.toml` into structured field definitions -- Define the field type system (`string`, `string[]`, `date`, `boolean`, `integer`, `float`, `enum`) +- Define the field type system (`string`, `string[]`, `date`, `boolean`, + `integer`, `float`, `enum`) - Infer field types from observed YAML/TOML/JSON values - Provide path-scoped validation via `allowed` and `required` glob patterns - Infer `allowed`/`required` patterns from file observations (tree inference) @@ -84,7 +88,9 @@ impl FieldDef { #### TOML defaults -When parsing from TOML, `allowed` defaults to `["**"]` (field allowed everywhere) and `required` defaults to `[]` (field not required anywhere). This makes the common case minimal: +When parsing from TOML, `allowed` defaults to `["**"]` (field allowed +everywhere) and `required` defaults to `[]` (field not required anywhere). This +makes the common case minimal: ```toml [[fields.field]] @@ -95,7 +101,8 @@ type = "string" #### Invariant: `required ⊆ allowed` -A field cannot be required somewhere it isn't allowed. If `required` is non-empty but `allowed` is empty, schema validation fails. +A field cannot be required somewhere it isn't allowed. If `required` is +non-empty but `allowed` is empty, schema validation fails. ### `Schema` @@ -139,7 +146,8 @@ impl FromStr for Schema { - `pattern` is a valid regex (only for `string`/`date` types) - `allowed` and `required` are valid glob patterns - `required ⊆ allowed` (required non-empty implies allowed non-empty) -- Unknown top-level sections are silently ignored (allows `mfv.toml` and `mdvs.toml` to share format) +- Unknown top-level sections are silently ignored (allows `mfv.toml` and + `mdvs.toml` to share format) ### `SchemaError` @@ -169,7 +177,8 @@ struct FieldInfo { ### `FieldPaths` -Inferred `allowed` and `required` patterns for a single field. Output of tree inference. +Inferred `allowed` and `required` patterns for a single field. Output of tree +inference. ```rust struct FieldPaths { @@ -227,7 +236,9 @@ impl LockFile { ## Type Inference -When `type` is not explicitly set in TOML, the type is inferred from observed values during `mfv init`. The inference logic lives in this crate so both tools use the same rules. +When `type` is not explicitly set in TOML, the type is inferred from observed +values during `mfv init`. The inference logic lives in this crate so both tools +use the same rules. ### `infer_type` Function @@ -235,19 +246,22 @@ When `type` is not explicitly set in TOML, the type is inferred from observed va fn infer_type(value: &serde_json::Value) -> FieldType ``` -Takes a single JSON value (converted from YAML/TOML frontmatter by `gray_matter`). Returns the inferred `FieldType`. +Takes a single JSON value (converted from YAML/TOML frontmatter by +`gray_matter`). Returns the inferred `FieldType`. -| Value | Inferred Type | -|---|---| -| JSON boolean | `Boolean` | -| JSON integer | `Integer` | -| JSON float | `Float` | -| JSON string matching YYYY-MM-DD | `Date` | -| JSON string (other) | `String` | -| JSON array | `StringArray` | -| Anything else | `String` | +| Value | Inferred Type | +| ------------------------------- | ------------- | +| JSON boolean | `Boolean` | +| JSON integer | `Integer` | +| JSON float | `Float` | +| JSON string matching YYYY-MM-DD | `Date` | +| JSON string (other) | `String` | +| JSON array | `StringArray` | +| Anything else | `String` | -**Mixed types:** When a field has different types across files, `discover_fields` picks the most common type. The user can override with an explicit `type` in the config. +**Mixed types:** When a field has different types across files, +`discover_fields` picks the most common type. The user can override with an +explicit `type` in the config. --- @@ -261,11 +275,14 @@ fn discover_fields( ) -> Vec ``` -Takes `(relative_path, frontmatter)` pairs. For each field found across all files, tracks: +Takes `(relative_path, frontmatter)` pairs. For each field found across all +files, tracks: + - The most common inferred type (majority vote) - Which files contain the field -Returns `Vec` sorted by frequency (descending), then name (ascending). +Returns `Vec` sorted by frequency (descending), then name +(ascending). --- @@ -279,35 +296,43 @@ fn infer_field_paths( ) -> BTreeMap ``` -Given a flat list of `(file_path, set_of_fields)`, infers `allowed` and `required` glob patterns for each field by building a directory tree and walking it. +Given a flat list of `(file_path, set_of_fields)`, infers `allowed` and +`required` glob patterns for each field by building a directory tree and walking +it. -See [Workflow: Inference](../../30-workflows/inference.md) for the full algorithm specification. +See [Workflow: Inference](../../30-workflows/inference.md) for the full +algorithm specification. Key behaviors: + - Leaf nodes (direct files) emit `*` (shallow) patterns - Directory nodes emit `**` (recursive) patterns via collapse - Collapse upgrades `*` to `**` when a directory confirms the claim -- `required` only comes from directory-level `all` sets (not leaf initialization) +- `required` only comes from directory-level `all` sets (not leaf + initialization) --- ## Dependencies -| Crate | Purpose | -|---|---| -| `serde` + `toml` | Parse TOML config | -| `serde_json` | Type inference from JSON values (via gray_matter) | -| `regex` | Compile and validate `pattern` rules | -| `globset` | Compile and validate `allowed`/`required` globs; path matching | -| `indextree` | Arena-backed tree for inference algorithm | -| `chrono` | Date string validation | +| Crate | Purpose | +| ---------------- | -------------------------------------------------------------- | +| `serde` + `toml` | Parse TOML config | +| `serde_json` | Type inference from JSON values (via gray_matter) | +| `regex` | Compile and validate `pattern` rules | +| `globset` | Compile and validate `allowed`/`required` globs; path matching | +| `indextree` | Arena-backed tree for inference algorithm | +| `chrono` | Date string validation | --- ## Related Documents -- [Terminology](../../01-terminology.md) — canonical definitions for field, field type -- [Configuration](../../40-configuration/frontmatter-toml.md) — file format this crate parses -- [Workflow: Inference](../../30-workflows/inference.md) — tree inference algorithm +- [Terminology](../../01-terminology.md) — canonical definitions for field, + field type +- [Configuration](../../40-configuration/frontmatter-toml.md) — file format this + crate parses +- [Workflow: Inference](../../30-workflows/inference.md) — tree inference + algorithm - [Crate: mfv](../mfv/spec.md) — validation engine that consumes this crate - [Crate: mdvs](../mdvs/spec.md) — search tool that consumes this crate diff --git a/docs/spec/archive/10-crates/mdvs/spec.md b/docs/spec/archive/10-crates/mdvs/spec.md index 1ac4d27..2618b36 100644 --- a/docs/spec/archive/10-crates/mdvs/spec.md +++ b/docs/spec/archive/10-crates/mdvs/spec.md @@ -2,19 +2,25 @@ **Status: DRAFT** -**Cross-references:** [Terminology](../../01-terminology.md) | [Crate: mdvs-schema](../mdvs-schema/spec.md) | [Crate: mfv](../mfv/spec.md) | [Storage Schema](../../20-storage/schema.md) +**Cross-references:** [Terminology](../../01-terminology.md) | +[Crate: mdvs-schema](../mdvs-schema/spec.md) | [Crate: mfv](../mfv/spec.md) | +[Storage Schema](../../20-storage/schema.md) --- ## Overview -Full semantic search CLI binary. Superset of `mfv` — does everything `mfv` does (field discovery, schema validation) plus frontmatter content querying and vector search over file contents. Depends on `mdvs-schema` and `mfv`. +Full semantic search CLI binary. Superset of `mfv` — does everything `mfv` does +(field discovery, schema validation) plus frontmatter content querying and +vector search over file contents. Depends on `mdvs-schema` and `mfv`. -**Architecture:** DataFusion (pure Rust SQL on Arrow) for querying, compressed Parquet files in `.mdvs/` for persistence. +**Architecture:** DataFusion (pure Rust SQL on Arrow) for querying, compressed +Parquet files in `.mdvs/` for persistence. **Responsibilities:** -- Initialize vault: discover fields, infer schema, download model, write config + lock +- Initialize vault: discover fields, infer schema, download model, write + config + lock - Build artifact: incremental ingestion, chunking, embedding, Parquet output - Search: embed query, cosine distance, note-level ranking - Model identity management and mismatch detection @@ -45,24 +51,27 @@ mdvs init [path] [--model ] [--glob ] [--config ] [--force] [--dry-run] [--ignore-bare-files] ``` -Subsumes `mfv init`. Discovers fields, infers types and allowed/required patterns via tree inference, writes config and lock, downloads the embedding model. +Subsumes `mfv init`. Discovers fields, infers types and allowed/required +patterns via tree inference, writes config and lock, downloads the embedding +model. **Flags:** -| Flag | Default | Description | -|---|---|---| -| `[path]` | `.` | Directory to scan | -| `--model ` | `minishlab/potion-multilingual-128M` | HuggingFace model ID | -| `--glob ` | `**` | File matching glob (path scope, `.md` hardcoded) | -| `--config ` | `mdvs.toml` | Output config file path | -| `--force` | off | Overwrite existing config and lock | -| `--dry-run` | off | Print discovery table only, write nothing | -| `--ignore-bare-files` | off | Exclude files without frontmatter from inference | +| Flag | Default | Description | +| --------------------- | ------------------------------------ | ------------------------------------------------ | +| `[path]` | `.` | Directory to scan | +| `--model ` | `minishlab/potion-multilingual-128M` | HuggingFace model ID | +| `--glob ` | `**` | File matching glob (path scope, `.md` hardcoded) | +| `--config ` | `mdvs.toml` | Output config file path | +| `--force` | off | Overwrite existing config and lock | +| `--dry-run` | off | Print discovery table only, write nothing | +| `--ignore-bare-files` | off | Exclude files without frontmatter from inference | **Steps:** 1. Scan directory, discover fields via `mdvs_schema::discover_fields` -2. Infer types and allowed/required patterns via `mdvs_schema::infer_field_paths` +2. Infer types and allowed/required patterns via + `mdvs_schema::infer_field_paths` 3. Display frequency table to stderr 4. Write `mdvs.toml` (field schema + `[model]` section) 5. Write `mdvs.lock` (field observations + file hashes) @@ -78,12 +87,14 @@ See [Workflow: Init](../../30-workflows/init.md). mdvs build [--full] ``` -Scans the directory, processes files into the `.mdvs/` artifact. Incremental by default — only changed files are reprocessed. Implicitly refreshes the lock before building (like `cargo build` updates `Cargo.lock`). +Scans the directory, processes files into the `.mdvs/` artifact. Incremental by +default — only changed files are reprocessed. Implicitly refreshes the lock +before building (like `cargo build` updates `Cargo.lock`). **Flags:** -| Flag | Description | -|---|---| +| Flag | Description | +| -------- | ------------------------------------------------------- | | `--full` | Clean rebuild: remove `.mdvs/` and reprocess everything | See [Workflow: Build](../../30-workflows/build.md). @@ -95,29 +106,30 @@ mdvs search [--where ] [-n ] [--format ] [--chunks] [--build] [--no-build] ``` -Embeds the query, computes cosine distance against all chunks, returns results ranked at the note level (or chunk level with `--chunks`). +Embeds the query, computes cosine distance against all chunks, returns results +ranked at the note level (or chunk level with `--chunks`). **Flags:** -| Flag | Default | Description | -|---|---|---| -| `--where ` | — | DataFusion SQL WHERE clause on files table | -| `-n ` | 10 | Number of results | -| `--format ` | `table` | Output format: `table`, `json`, `paths` | -| `--chunks` | off | Show chunk-level results instead of note-level grouping | -| `--build` | — | Force build before search (overrides `on_stale` config) | -| `--no-build` | — | Never auto-build (overrides `on_stale` config) | +| Flag | Default | Description | +| ------------------ | ------- | ------------------------------------------------------- | +| `--where ` | — | DataFusion SQL WHERE clause on files table | +| `-n ` | 10 | Number of results | +| `--format ` | `table` | Output format: `table`, `json`, `paths` | +| `--chunks` | off | Show chunk-level results instead of note-level grouping | +| `--build` | — | Force build before search (overrides `on_stale` config) | +| `--no-build` | — | Never auto-build (overrides `on_stale` config) | **Auto-build behavior:** -| Config `on_stale` | `--build` | `--no-build` | Result | -|---|---|---|---| -| `auto` | — | — | Build if stale | -| `auto` | — | yes | Skip build | -| `strict` | — | — | Error if stale | -| `strict` | yes | — | Build if stale | -| any | yes | — | Always build | -| any | — | yes | Never build | +| Config `on_stale` | `--build` | `--no-build` | Result | +| ----------------- | --------- | ------------ | -------------- | +| `auto` | — | — | Build if stale | +| `auto` | — | yes | Skip build | +| `strict` | — | — | Error if stale | +| `strict` | yes | — | Build if stale | +| any | yes | — | Always build | +| any | — | yes | Never build | See [Workflow: Search](../../30-workflows/search.md). @@ -127,7 +139,8 @@ See [Workflow: Search](../../30-workflows/search.md). mdvs update [--dir ] [--config ] ``` -Re-scans the directory and refreshes the lock file. Same as `mfv update` but for `mdvs.toml`/`mdvs.lock`. Does not modify config. +Re-scans the directory and refreshes the lock file. Same as `mfv update` but for +`mdvs.toml`/`mdvs.lock`. Does not modify config. ### `mdvs check` @@ -135,9 +148,11 @@ Re-scans the directory and refreshes the lock file. Same as `mfv update` but for mdvs check [--dir ] [--schema ] [--format ] ``` -Delegates to `mfv::validate`. Validates all matching markdown files against the field schema. Convenience command so users don't need a separate `mfv` binary. +Delegates to `mfv::validate`. Validates all matching markdown files against the +field schema. Convenience command so users don't need a separate `mfv` binary. -**Exit codes:** 0 = all valid, 1 = validation errors found, 2 = config/runtime error. +**Exit codes:** 0 = all valid, 1 = validation errors found, 2 = config/runtime +error. ### `mdvs clean` @@ -153,7 +168,8 @@ Removes the `.mdvs/` artifact directory. Does not touch config or lock files. mdvs info ``` -Displays: vault path, artifact size, file count, chunk count, model ID/dimension/revision, last build timestamp. +Displays: vault path, artifact size, file count, chunk count, model +ID/dimension/revision, last build timestamp. --- @@ -270,47 +286,53 @@ See [Workflow: Model Mismatch](../../30-workflows/model-mismatch.md). Configured via `on_stale` in `mdvs.toml` `[behavior]` section: -| Mode | Behavior | -|---|---| -| `auto` | Run incremental build before search if stale (default) | -| `strict` | Error if any files have changed since last build | +| Mode | Behavior | +| -------- | ------------------------------------------------------ | +| `auto` | Run incremental build before search if stale (default) | +| `strict` | Error if any files have changed since last build | -In `auto` mode, `mdvs search` transparently runs the equivalent of `mdvs build` first, so results are always fresh. The overhead is minimal for unchanged vaults (just a hash comparison). +In `auto` mode, `mdvs search` transparently runs the equivalent of `mdvs build` +first, so results are always fresh. The overhead is minimal for unchanged vaults +(just a hash comparison). -CLI overrides: `--build` forces a build regardless of config, `--no-build` skips it. +CLI overrides: `--build` forces a build regardless of config, `--no-build` skips +it. --- ## Dependencies -| Crate | Purpose | -|---|---| -| `mdvs-schema` | Field definitions, type system, TOML parsing | -| `mfv` | Validation engine (library dependency) | -| `datafusion` | SQL query engine on Arrow | -| `parquet` | Parquet file I/O | -| `arrow` | Arrow columnar data types | -| `model2vec-rs` | Static embedding inference | -| `gray_matter` | Frontmatter extraction | -| `text-splitter` (markdown) | Semantic chunking | -| `pulldown-cmark` | Markdown → plain text | -| `clap` | CLI parsing | -| `anyhow` | Error handling | -| `walkdir` | Filesystem traversal | -| `indicatif` | Progress bars | -| `xxhash-rust` or `blake3` | Content hashing | -| `serde_json` | JSON output format | +| Crate | Purpose | +| -------------------------- | -------------------------------------------- | +| `mdvs-schema` | Field definitions, type system, TOML parsing | +| `mfv` | Validation engine (library dependency) | +| `datafusion` | SQL query engine on Arrow | +| `parquet` | Parquet file I/O | +| `arrow` | Arrow columnar data types | +| `model2vec-rs` | Static embedding inference | +| `gray_matter` | Frontmatter extraction | +| `text-splitter` (markdown) | Semantic chunking | +| `pulldown-cmark` | Markdown → plain text | +| `clap` | CLI parsing | +| `anyhow` | Error handling | +| `walkdir` | Filesystem traversal | +| `indicatif` | Progress bars | +| `xxhash-rust` or `blake3` | Content hashing | +| `serde_json` | JSON output format | --- ## Related Documents - [Terminology](../../01-terminology.md) -- [Storage Schema](../../20-storage/schema.md) — Parquet file schemas and type mappings +- [Storage Schema](../../20-storage/schema.md) — Parquet file schemas and type + mappings - [Workflow: Init](../../30-workflows/init.md) — full init flow - [Workflow: Build](../../30-workflows/build.md) — incremental build pipeline - [Workflow: Search](../../30-workflows/search.md) — query, rank, display -- [Workflow: Model Loading](../../30-workflows/model-loading.md) — format detection, universal loader -- [Workflow: Model Mismatch](../../30-workflows/model-mismatch.md) — identity checks +- [Workflow: Model Loading](../../30-workflows/model-loading.md) — format + detection, universal loader +- [Workflow: Model Mismatch](../../30-workflows/model-mismatch.md) — identity + checks - [Configuration: Field Schema](../../40-configuration/frontmatter-toml.md) - [Configuration: mdvs.toml](../../40-configuration/mdvs-toml.md) diff --git a/docs/spec/archive/10-crates/mfv/spec.md b/docs/spec/archive/10-crates/mfv/spec.md index cbbeb52..199b42b 100644 --- a/docs/spec/archive/10-crates/mfv/spec.md +++ b/docs/spec/archive/10-crates/mfv/spec.md @@ -2,21 +2,26 @@ **Status: DRAFT** -**Cross-references:** [Terminology](../../01-terminology.md) | [Crate: mdvs-schema](../mdvs-schema/spec.md) | [Configuration](../../40-configuration/frontmatter-toml.md) +**Cross-references:** [Terminology](../../01-terminology.md) | +[Crate: mdvs-schema](../mdvs-schema/spec.md) | +[Configuration](../../40-configuration/frontmatter-toml.md) --- ## Overview -Standalone frontmatter validation library and CLI binary (~2MB). No embeddings, no search. Independently publishable on crates.io as `mfv`. +Standalone frontmatter validation library and CLI binary (~2MB). No embeddings, +no search. Independently publishable on crates.io as `mfv`. -**Target users:** bloggers, documentation maintainers, CI pipelines — anyone with markdown + frontmatter who wants linting. +**Target users:** bloggers, documentation maintainers, CI pipelines — anyone +with markdown + frontmatter who wants linting. **Responsibilities:** - Scan markdown files and extract frontmatter via `gray_matter` - Validate frontmatter against a field schema (`mfv.toml` / `mdvs.toml`) -- Generate a field schema by scanning, inferring types, and inferring allowed/required patterns +- Generate a field schema by scanning, inferring types, and inferring + allowed/required patterns - Report diagnostics in human-readable, JSON, or GitHub Actions format - Provide a library API (`mfv::validate`) consumed by `mdvs` @@ -44,35 +49,42 @@ COMMANDS: mfv init [--dir ] [--glob ] [--config ] [--force] [--dry-run] ``` -Scans markdown files, discovers frontmatter fields, infers types and allowed/required patterns via tree inference, and writes `mfv.toml` (schema) and `mfv.lock` (per-file observations). +Scans markdown files, discovers frontmatter fields, infers types and +allowed/required patterns via tree inference, and writes `mfv.toml` (schema) and +`mfv.lock` (per-file observations). **Flags:** -| Flag | Default | Description | -|---|---|---| -| `--dir ` | `.` | Directory to scan | -| `--glob ` | `**` | File matching glob | -| `--config ` | `mfv.toml` | Output config file path | -| `--force` | off | Overwrite existing config and lock | -| `--dry-run` | off | Print discovery table only, write nothing | -| `--ignore-bare-files` | off | Exclude files without frontmatter from analysis | +| Flag | Default | Description | +| --------------------- | ---------- | ----------------------------------------------- | +| `--dir ` | `.` | Directory to scan | +| `--glob ` | `**` | File matching glob | +| `--config ` | `mfv.toml` | Output config file path | +| `--force` | off | Overwrite existing config and lock | +| `--dry-run` | off | Print discovery table only, write nothing | +| `--ignore-bare-files` | off | Exclude files without frontmatter from analysis | **Flow:** -1. Walk directory with glob filter (only `.md` files are processed — the glob scopes which directories to scan, not which file extensions). When `--ignore-bare-files` is set, files without frontmatter are excluded before inference. +1. Walk directory with glob filter (only `.md` files are processed — the glob + scopes which directories to scan, not which file extensions). When + `--ignore-bare-files` is set, files without frontmatter are excluded before + inference. 2. Extract frontmatter from each file via `gray_matter` 3. Discover fields and infer types via `mdvs_schema::discover_fields` -4. Build per-file field observations and run `mdvs_schema::infer_field_paths` (tree inference) +4. Build per-file field observations and run `mdvs_schema::infer_field_paths` + (tree inference) 5. Combine: `FieldDef` = inferred type + inferred `allowed`/`required` patterns 6. Display frequency table to stderr -7. Write `mfv.toml` (schema with patterns) and `mfv.lock` (per-file observations) +7. Write `mfv.toml` (schema with patterns) and `mfv.lock` (per-file + observations) **Exit codes:** -| Code | Meaning | -|---|---| -| 0 | Config and lock written (or dry-run completed) | -| 2 | Config/IO error (config exists without `--force`, directory not found, no files found, etc.) | +| Code | Meaning | +| ---- | -------------------------------------------------------------------------------------------- | +| 0 | Config and lock written (or dry-run completed) | +| 2 | Config/IO error (config exists without `--force`, directory not found, no files found, etc.) | ### `mfv update` @@ -80,24 +92,27 @@ Scans markdown files, discovers frontmatter fields, infers types and allowed/req mfv update [--dir ] [--config ] ``` -Re-scans the directory, discovers fields, and refreshes the lock file. Does not modify config. Analogous to `cargo update`. +Re-scans the directory, discovers fields, and refreshes the lock file. Does not +modify config. Analogous to `cargo update`. **Flags:** -| Flag | Default | Description | -|---|---|---| -| `--dir ` | `.` | Directory to scan | -| `--config ` | auto | Path to config file (auto-discover `mfv.toml` / `mdvs.toml`) | +| Flag | Default | Description | +| ----------------- | ------- | ------------------------------------------------------------ | +| `--dir ` | `.` | Directory to scan | +| `--config ` | auto | Path to config file (auto-discover `mfv.toml` / `mdvs.toml`) | **Flow:** 1. Find existing config (`--config` or auto-discover `mfv.toml` → `mdvs.toml`) 2. Load config to extract glob pattern -3. Scan directory, discover fields, infer patterns (identical to init steps 1-4; only `.md` files are processed) +3. Scan directory, discover fields, infer patterns (identical to init steps 1-4; + only `.md` files are processed) 4. Display frequency table to stderr 5. Write lock file (overwrites existing) -**Exit codes:** 0 = success, 2 = config/IO error (missing config, bad directory, etc.) +**Exit codes:** 0 = success, 2 = config/IO error (missing config, bad directory, +etc.) ### `mfv check` @@ -112,21 +127,23 @@ Options: Validates all matching markdown files against the field schema. -**Config discovery** (when `--schema` is not given): `mfv.toml` → `mdvs.toml` → error. +**Config discovery** (when `--schema` is not given): `mfv.toml` → `mdvs.toml` → +error. **Exit codes:** -| Code | Meaning | -|---|---| -| 0 | All files valid | -| 1 | Validation errors found | -| 2 | Schema/config error (bad TOML, missing file, directory not found, etc.) | +| Code | Meaning | +| ---- | ----------------------------------------------------------------------- | +| 0 | All files valid | +| 1 | Validation errors found | +| 2 | Schema/config error (bad TOML, missing file, directory not found, etc.) | --- ## Library API -The `mfv` crate exposes a library API so `mdvs` can delegate validation without spawning a subprocess. +The `mfv` crate exposes a library API so `mdvs` can delegate validation without +spawning a subprocess. ### Modules @@ -148,7 +165,8 @@ pub fn validate( ) -> Vec ``` -Validates scanned files against a schema. Returns a list of diagnostics (empty = all valid). +Validates scanned files against a schema. Returns a list of diagnostics (empty = +all valid). ### `ScannedFile` @@ -189,19 +207,24 @@ pub enum DiagnosticKind { ## Validation Rules -All rules are defined in the TOML config via `mdvs-schema`. The `mfv` crate executes them. +All rules are defined in the TOML config via `mdvs-schema`. The `mfv` crate +executes them. ### Rule Evaluation Order For each file: -1. Determine which fields apply to this file via `schema.rules_for_path(rel_path)` — filters by `allowed` patterns. -2. For each applicable field: - a. If `rule.is_required_at(rel_path)` and field is absent → `MissingRequired`. - b. If field is present, check type compatibility → `WrongType` if wrong. - c. If `pattern` is set and value is a string, check regex → `PatternMismatch`. - d. If `values` is set (enum), check membership → `InvalidEnum`. -3. **Allowed enforcement:** For each key in the file's frontmatter, check that some schema field with that name has `is_allowed_at(rel_path)`. If no match → `NotAllowed`. This catches both fields with restricted `allowed` patterns appearing outside their scope and fields not defined in the schema at all. +1. Determine which fields apply to this file via + `schema.rules_for_path(rel_path)` — filters by `allowed` patterns. +2. For each applicable field: a. If `rule.is_required_at(rel_path)` and field is + absent → `MissingRequired`. b. If field is present, check type compatibility + → `WrongType` if wrong. c. If `pattern` is set and value is a string, check + regex → `PatternMismatch`. d. If `values` is set (enum), check membership → + `InvalidEnum`. +3. **Allowed enforcement:** For each key in the file's frontmatter, check that + some schema field with that name has `is_allowed_at(rel_path)`. If no match → + `NotAllowed`. This catches both fields with restricted `allowed` patterns + appearing outside their scope and fields not defined in the schema at all. ### Path-Scoped Rules @@ -215,7 +238,9 @@ allowed = ["blog/**"] required = ["blog/**"] ``` -A file at `blog/my-post.md` must have `status` (it's both allowed and required there). A file at `notes/random.md` is not checked for `status` at all (not in `allowed`). +A file at `blog/my-post.md` must have `status` (it's both allowed and required +there). A file at `notes/random.md` is not checked for `status` at all (not in +`allowed`). --- @@ -255,24 +280,28 @@ Enables inline annotations in GitHub PR diffs. ## Dependencies -| Crate | Purpose | -|---|---| +| Crate | Purpose | +| ------------- | ------------------------------------------------------------------ | | `mdvs-schema` | Field definitions, type system, TOML parsing, discovery, inference | -| `gray_matter` | Frontmatter extraction from markdown files | -| `walkdir` | Recursive directory traversal | -| `globset` | Glob pattern matching | -| `regex` | Regex validation for `pattern` rules | -| `clap` | CLI argument parsing | -| `anyhow` | Error handling | -| `serde_json` | JSON output format | -| `chrono` | Timestamp generation for lock file | +| `gray_matter` | Frontmatter extraction from markdown files | +| `walkdir` | Recursive directory traversal | +| `globset` | Glob pattern matching | +| `regex` | Regex validation for `pattern` rules | +| `clap` | CLI argument parsing | +| `anyhow` | Error handling | +| `serde_json` | JSON output format | +| `chrono` | Timestamp generation for lock file | --- ## Related Documents -- [Terminology](../../01-terminology.md) — canonical definitions for frontmatter, field, field type -- [Crate: mdvs-schema](../mdvs-schema/spec.md) — types and parsing consumed by this crate -- [Configuration](../../40-configuration/frontmatter-toml.md) — schema file format +- [Terminology](../../01-terminology.md) — canonical definitions for + frontmatter, field, field type +- [Crate: mdvs-schema](../mdvs-schema/spec.md) — types and parsing consumed by + this crate +- [Configuration](../../40-configuration/frontmatter-toml.md) — schema file + format - [Workflow: Init](../../30-workflows/init.md) — init flow -- [Workflow: Inference](../../30-workflows/inference.md) — tree inference algorithm +- [Workflow: Inference](../../30-workflows/inference.md) — tree inference + algorithm diff --git a/docs/spec/archive/20-storage/schema.md b/docs/spec/archive/20-storage/schema.md index 991c67d..47ce3d5 100644 --- a/docs/spec/archive/20-storage/schema.md +++ b/docs/spec/archive/20-storage/schema.md @@ -2,15 +2,21 @@ **Status: DRAFT** -**Cross-references:** [Terminology](../01-terminology.md) | [Crate: mdvs](../10-crates/mdvs/spec.md) | [Crate: mdvs-schema](../10-crates/mdvs-schema/spec.md) +**Cross-references:** [Terminology](../01-terminology.md) | +[Crate: mdvs](../10-crates/mdvs/spec.md) | +[Crate: mdvs-schema](../10-crates/mdvs-schema/spec.md) --- ## Overview -The mdvs artifact is a `.mdvs/` directory at the root of the vault, containing compressed Parquet files. DataFusion (pure Rust SQL engine on Arrow) registers these files as tables for querying. The directory is co-located with the data, portable (move the vault, the index follows), and `.gitignore`-able. +The mdvs artifact is a `.mdvs/` directory at the root of the vault, containing +compressed Parquet files. DataFusion (pure Rust SQL engine on Arrow) registers +these files as tables for querying. The directory is co-located with the data, +portable (move the vault, the index follows), and `.gitignore`-able. -Artifact metadata (model identity, build timestamps) lives in `mdvs.lock`, not in `.mdvs/`. The artifact directory contains only data files. +Artifact metadata (model identity, build timestamps) lives in `mdvs.lock`, not +in `.mdvs/`. The artifact directory contains only data files. --- @@ -32,50 +38,65 @@ vault/ ### `files.parquet` -One row per markdown file. Fixed schema — all frontmatter is stored as a single JSON column. +One row per markdown file. Fixed schema — all frontmatter is stored as a single +JSON column. -| Column | Arrow Type | Description | -|---|---|---| -| `file_id` | `Utf8` | UUID v4 (primary key) | -| `filename` | `Utf8` | Relative path from vault root | -| `frontmatter` | `Utf8` | Full frontmatter as JSON string | -| `content_hash` | `Utf8` | xxh3 hash of full file content | -| `built_at` | `Timestamp(Microsecond, None)` | When this file was last processed | +| Column | Arrow Type | Description | +| -------------- | ------------------------------ | --------------------------------- | +| `file_id` | `Utf8` | UUID v4 (primary key) | +| `filename` | `Utf8` | Relative path from vault root | +| `frontmatter` | `Utf8` | Full frontmatter as JSON string | +| `content_hash` | `Utf8` | xxh3 hash of full file content | +| `built_at` | `Timestamp(Microsecond, None)` | When this file was last processed | -**`file_id`:** UUID v4, generated at build time. Decouples identity from path — enables future rename detection without orphaning chunks. +**`file_id`:** UUID v4, generated at build time. Decouples identity from path — +enables future rename detection without orphaning chunks. -**`frontmatter`:** JSON string containing all frontmatter fields. Files without frontmatter get NULL. Frontmatter filtering (e.g., `WHERE tags LIKE '%rust%'`) uses JSON extraction functions in DataFusion. +**`frontmatter`:** JSON string containing all frontmatter fields. Files without +frontmatter get NULL. Frontmatter filtering (e.g., `WHERE tags LIKE '%rust%'`) +uses JSON extraction functions in DataFusion. -**`content_hash`:** Used for incremental builds. Compare against `mdvs.lock` `[[file]]` entries to detect changes. +**`content_hash`:** Used for incremental builds. Compare against `mdvs.lock` +`[[file]]` entries to detect changes. ### `chunks.parquet` One row per semantic chunk of a note. -| Column | Arrow Type | Description | -|---|---|---| -| `chunk_id` | `Utf8` | UUID v4 | -| `file_id` | `Utf8` | FK to `files.parquet` | -| `chunk_index` | `Int32` | 0-based position within the file | -| `start_line` | `Int32` | Start line number in file (1-based) | -| `end_line` | `Int32` | End line number in file (1-based) | -| `embedding` | `FixedSizeList(N)` | N = model dimension (e.g., 256) | +| Column | Arrow Type | Description | +| ------------- | --------------------------- | ----------------------------------- | +| `chunk_id` | `Utf8` | UUID v4 | +| `file_id` | `Utf8` | FK to `files.parquet` | +| `chunk_index` | `Int32` | 0-based position within the file | +| `start_line` | `Int32` | Start line number in file (1-based) | +| `end_line` | `Int32` | End line number in file (1-based) | +| `embedding` | `FixedSizeList(N)` | N = model dimension (e.g., 256) | -**`chunk_id`:** UUID v4. Stable identity that survives re-chunking — in future chunk-level hashing (v0.4+), unchanged chunks keep their ID and embedding even if their position shifts. +**`chunk_id`:** UUID v4. Stable identity that survives re-chunking — in future +chunk-level hashing (v0.4+), unchanged chunks keep their ID and embedding even +if their position shifts. -**`chunk_index`:** Positional ordering within the file. Recomputed on every rebuild. Not an identity — use `chunk_id` for stable references. +**`chunk_index`:** Positional ordering within the file. Recomputed on every +rebuild. Not an identity — use `chunk_id` for stable references. -**`start_line` / `end_line`:** 1-based line numbers matching editor display. Used by `--snippets` to read chunk text directly from the original file. In rare cases where the splitter falls back to character-level splitting mid-line, two consecutive chunks may share a line number — this is acceptable. +**`start_line` / `end_line`:** 1-based line numbers matching editor display. +Used by `--snippets` to read chunk text directly from the original file. In rare +cases where the splitter falls back to character-level splitting mid-line, two +consecutive chunks may share a line number — this is acceptable. -**`embedding`:** `FixedSizeList(N)` where N is determined at build time from the model's output dimension. +**`embedding`:** `FixedSizeList(N)` where N is determined at build time +from the model's output dimension. -No `plain_text` stored — model changes require re-reading files from disk to re-chunk and re-embed. This keeps the artifact lightweight. +No `plain_text` stored — model changes require re-reading files from disk to +re-chunk and re-embed. This keeps the artifact lightweight. --- ## Compression -Parquet files use built-in compression. The default is snappy (fast, reasonable ratio). Zstd is an alternative for better compression at slightly higher CPU cost. The compression codec is an implementation detail, not user-configurable. +Parquet files use built-in compression. The default is snappy (fast, reasonable +ratio). Zstd is an alternative for better compression at slightly higher CPU +cost. The compression codec is an implementation detail, not user-configurable. --- @@ -92,13 +113,15 @@ ctx.register_parquet("chunks", ".mdvs/chunks.parquet", opts).await?; Cosine distance is **not** computed via SQL. Instead: -1. Load `chunks.parquet` embedding column as an Arrow `FixedSizeList` array +1. Load `chunks.parquet` embedding column as an Arrow `FixedSizeList` + array 2. Compute cosine distance in Rust (vectorized over the Arrow array) 3. Append the distance as a new `Float64` column to the RecordBatch 4. Register the enriched RecordBatch as a DataFusion table 5. Use DataFusion SQL for JOIN, GROUP BY, ORDER BY, LIMIT, WHERE -This avoids the complexity of DataFusion UDFs while keeping the hot path (distance computation) in pure Rust. +This avoids the complexity of DataFusion UDFs while keeping the hot path +(distance computation) in pure Rust. ### Note-Level Ranking @@ -114,7 +137,9 @@ ORDER BY distance LIMIT ?; ``` -Default search output is ranked file paths only. With `--snippets`, the best-matching chunk's `start_line`/`end_line` are used to read text from the original file. +Default search output is ranked file paths only. With `--snippets`, the +best-matching chunk's `start_line`/`end_line` are used to read text from the +original file. ### Chunk-Level Search @@ -135,19 +160,26 @@ LIMIT ?; ### Incremental Build Diff -Content hashes are stored in `mdvs.lock` `[[file]]` entries. The build pipeline compares these against freshly computed hashes from the filesystem to determine new/modified/deleted/unchanged files. +Content hashes are stored in `mdvs.lock` `[[file]]` entries. The build pipeline +compares these against freshly computed hashes from the filesystem to determine +new/modified/deleted/unchanged files. --- ## Vector Search -No ANN index initially — search performs brute-force cosine distance over all chunks. For typical vault sizes (< 50k chunks), this is fast enough. `hnsw_rs` is a future upgrade path for larger vaults. +No ANN index initially — search performs brute-force cosine distance over all +chunks. For typical vault sizes (< 50k chunks), this is fast enough. `hnsw_rs` +is a future upgrade path for larger vaults. --- ## Related Documents -- [Terminology](../01-terminology.md) — definitions for artifact, chunk, embedding, build -- [Crate: mdvs](../10-crates/mdvs/spec.md) — `storage` module that implements Parquet I/O +- [Terminology](../01-terminology.md) — definitions for artifact, chunk, + embedding, build +- [Crate: mdvs](../10-crates/mdvs/spec.md) — `storage` module that implements + Parquet I/O - [Workflow: Build](../30-workflows/build.md) — how data flows into these files -- [Workflow: Search](../30-workflows/search.md) — how queries execute against these files +- [Workflow: Search](../30-workflows/search.md) — how queries execute against + these files diff --git a/docs/spec/archive/30-workflows/build.md b/docs/spec/archive/30-workflows/build.md index 1ead0ba..bbe47b8 100644 --- a/docs/spec/archive/30-workflows/build.md +++ b/docs/spec/archive/30-workflows/build.md @@ -2,28 +2,34 @@ **Status: DRAFT** -**Cross-references:** [Terminology](../01-terminology.md) | [Crate: mdvs](../10-crates/mdvs/spec.md) | [Storage Schema](../20-storage/schema.md) +**Cross-references:** [Terminology](../01-terminology.md) | +[Crate: mdvs](../10-crates/mdvs/spec.md) | +[Storage Schema](../20-storage/schema.md) --- ## Overview -The build workflow processes markdown files into the `.mdvs/` artifact: extract frontmatter, split into chunks, compute embeddings, and write compressed Parquet files. Default mode is incremental — only changed files are reprocessed. Analogous to `cargo build`. +The build workflow processes markdown files into the `.mdvs/` artifact: extract +frontmatter, split into chunks, compute embeddings, and write compressed Parquet +files. Default mode is incremental — only changed files are reprocessed. +Analogous to `cargo build`. -The build implicitly refreshes the lock file before processing, like `cargo build` updates `Cargo.lock`. +The build implicitly refreshes the lock file before processing, like +`cargo build` updates `Cargo.lock`. --- ## Actors -| Actor | Role | -|---|---| -| **CLI** | Orchestrates the pipeline | -| **Filesystem** | Source of `.md` files | -| **gray_matter** | Frontmatter extraction | -| **text-splitter** | Semantic chunking | -| **pulldown-cmark** | Markdown → plain text | -| **model2vec-rs** | Embedding inference | +| Actor | Role | +| ---------------------- | ----------------------------- | +| **CLI** | Orchestrates the pipeline | +| **Filesystem** | Source of `.md` files | +| **gray_matter** | Frontmatter extraction | +| **text-splitter** | Semantic chunking | +| **pulldown-cmark** | Markdown → plain text | +| **model2vec-rs** | Embedding inference | | **Parquet/DataFusion** | Storage (write Parquet files) | --- @@ -86,22 +92,26 @@ sequenceDiagram ## Diff Logic -Content hashes in `mdvs.lock` `[[file]]` entries are compared against freshly computed hashes from the filesystem: +Content hashes in `mdvs.lock` `[[file]]` entries are compared against freshly +computed hashes from the filesystem: -| Category | Condition | Action | -|---|---|---| -| **New** | File on disk, not in lock | Process: extract + chunk + embed | -| **Modified** | File on disk, in lock, hash differs | Reprocess: extract + chunk + embed | -| **Deleted** | In lock, not on disk | Remove from artifact | +| Category | Condition | Action | +| ------------- | ----------------------------------- | ----------------------------------- | +| **New** | File on disk, not in lock | Process: extract + chunk + embed | +| **Modified** | File on disk, in lock, hash differs | Reprocess: extract + chunk + embed | +| **Deleted** | In lock, not on disk | Remove from artifact | | **Unchanged** | File on disk, in lock, hash matches | Carry forward from existing Parquet | -Content hash is computed over the full file content (frontmatter + body), not just the body. This means frontmatter-only changes (e.g., updating tags) also trigger reprocessing. +Content hash is computed over the full file content (frontmatter + body), not +just the body. This means frontmatter-only changes (e.g., updating tags) also +trigger reprocessing. --- ## Full Build Mode (`--full`) -Skips the diff phase. Removes existing `.mdvs/` and processes all files from scratch: +Skips the diff phase. Removes existing `.mdvs/` and processes all files from +scratch: 1. Delete `.mdvs/` directory 2. Process every file (extract, chunk, embed) @@ -113,32 +123,37 @@ Use after a model change or when the artifact is suspected to be inconsistent. ## Batch Processing -Embeddings are computed in batches for efficiency. `model2vec-rs` supports batch inference. The batch size is an implementation detail (e.g., 256 chunks per batch), not user-configurable. +Embeddings are computed in batches for efficiency. `model2vec-rs` supports batch +inference. The batch size is an implementation detail (e.g., 256 chunks per +batch), not user-configurable. -The progress bar (via `indicatif`) shows per-file progress during the extract+chunk phase and per-batch progress during embedding. +The progress bar (via `indicatif`) shows per-file progress during the +extract+chunk phase and per-batch progress during embedding. --- ## Edge Cases -| Case | Behavior | -|---|---| -| File with no frontmatter | Field columns = NULL, metadata = `{}`, body still chunked and embedded | -| File with unparseable frontmatter | Log warning, treat as no frontmatter (still build the full content) | -| Empty file | Skip: no content to chunk or embed | -| Very large file (>1MB) | Normal processing: `text-splitter` handles arbitrary sizes via cascading chunk splits | -| Binary file matched by glob | Skip: if not valid UTF-8, skip silently | -| Non-UTF8 filename | Skip with warning | +| Case | Behavior | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| File with no frontmatter | Field columns = NULL, metadata = `{}`, body still chunked and embedded | +| File with unparseable frontmatter | Log warning, treat as no frontmatter (still build the full content) | +| Empty file | Skip: no content to chunk or embed | +| Very large file (>1MB) | Normal processing: `text-splitter` handles arbitrary sizes via cascading chunk splits | +| Binary file matched by glob | Skip: if not valid UTF-8, skip silently | +| Non-UTF8 filename | Skip with warning | | File changes between hash computation and read | Race condition: content hash won't match on next build, triggering a reprocess. Self-correcting. | -| No `.mdvs/` directory | First build: create it | -| Config not found | Error: "mdvs.toml not found. Run `mdvs init` first." | +| No `.mdvs/` directory | First build: create it | +| Config not found | Error: "mdvs.toml not found. Run `mdvs init` first." | --- ## Related Documents -- [Terminology](../01-terminology.md) — definitions for chunk, plain text, content hash, build -- [Crate: mdvs](../10-crates/mdvs/spec.md) — `ingest`, `embed`, `storage` modules +- [Terminology](../01-terminology.md) — definitions for chunk, plain text, + content hash, build +- [Crate: mdvs](../10-crates/mdvs/spec.md) — `ingest`, `embed`, `storage` + modules - [Storage Schema](../20-storage/schema.md) — Parquet file schemas - [Workflow: Model Mismatch](model-mismatch.md) — identity check before building - [Workflow: Init](init.md) — must run before first build diff --git a/docs/spec/archive/30-workflows/inference.md b/docs/spec/archive/30-workflows/inference.md index 4596006..93bdb92 100644 --- a/docs/spec/archive/30-workflows/inference.md +++ b/docs/spec/archive/30-workflows/inference.md @@ -2,18 +2,24 @@ **Status: DRAFT** -**Cross-references:** [Terminology](../01-terminology.md) | [Crate: mdvs-schema](../10-crates/mdvs-schema/spec.md) | [Configuration](../40-configuration/frontmatter-toml.md) | [Workflow: Init](init.md) +**Cross-references:** [Terminology](../01-terminology.md) | +[Crate: mdvs-schema](../10-crates/mdvs-schema/spec.md) | +[Configuration](../40-configuration/frontmatter-toml.md) | +[Workflow: Init](init.md) --- ## Overview -Field inference takes a flat list of `(file_path, set_of_fields)` observations and produces, for each field, two sets of glob patterns: +Field inference takes a flat list of `(file_path, set_of_fields)` observations +and produces, for each field, two sets of glob patterns: -- **`allowed`** — where the field *may* appear (at least one file has it) -- **`required`** — where the field *must* appear (all files have it) +- **`allowed`** — where the field _may_ appear (at least one file has it) +- **`required`** — where the field _must_ appear (all files have it) -These patterns are written to the TOML config (`mfv.toml` / `mdvs.toml`) during `init`. The user can then hand-edit them for finer control. `check` validates files against these patterns. +These patterns are written to the TOML config (`mfv.toml` / `mdvs.toml`) during +`init`. The user can then hand-edit them for finer control. `check` validates +files against these patterns. ### Semantics @@ -22,7 +28,8 @@ These patterns are written to the TOML config (`mfv.toml` / `mdvs.toml`) during - `["blog/**"]` = all files at any depth under `blog/` - `["blog/*"]` = files directly in `blog/`, not in subdirectories -Invariant: `required ⊆ allowed` — you cannot require a field where it's not allowed. +Invariant: `required ⊆ allowed` — you cannot require a field where it's not +allowed. --- @@ -30,27 +37,37 @@ Invariant: `required ⊆ allowed` — you cannot require a field where it's not ### Definitions -Given a vault with files $F = \{f_1, \ldots, f_n\}$, each file $f_i$ has a set of frontmatter fields $\text{fields}(f_i) \subseteq \mathcal{F}$ where $\mathcal{F}$ is the universe of all observed field names. +Given a vault with files $F = \{f_1, \ldots, f_n\}$, each file $f_i$ has a set +of frontmatter fields $\text{fields}(f_i) \subseteq \mathcal{F}$ where +$\mathcal{F}$ is the universe of all observed field names. For a set of files $S \subseteq F$: -- $\text{all}(S) = \bigcap_{f \in S} \text{fields}(f)$ — fields present in **every** file -- $\text{any}(S) = \bigcup_{f \in S} \text{fields}(f)$ — fields present in **at least one** file +- $\text{all}(S) = \bigcap_{f \in S} \text{fields}(f)$ — fields present in + **every** file +- $\text{any}(S) = \bigcup_{f \in S} \text{fields}(f)$ — fields present in **at + least one** file - Invariant: $\text{all}(S) \subseteq \text{any}(S)$ ### The directory tree Files live in a directory hierarchy. We model this as a tree: -- **Internal nodes** = directories. A directory is *always* an internal node, even if it contains files directly. Directories never appear as leaves. -- **Leaf nodes** = file-set aggregates. The set of files directly in a directory, represented as a single leaf. A directory with 3 files gets one leaf child holding the aggregate of those 3 files. +- **Internal nodes** = directories. A directory is _always_ an internal node, + even if it contains files directly. Directories never appear as leaves. +- **Leaf nodes** = file-set aggregates. The set of files directly in a + directory, represented as a single leaf. A directory with 3 files gets one + leaf child holding the aggregate of those 3 files. - Empty directories (no files, no non-empty subdirectories) are excluded. -This distinction is critical. A directory may have both a file-set leaf (its direct files) and subdirectory children. These are siblings in the tree, treated uniformly during merge and collapse. +This distinction is critical. A directory may have both a file-set leaf (its +direct files) and subdirectory children. These are siblings in the tree, treated +uniformly during merge and collapse. #### Example tree -Given files: `blog/post1.md`, `blog/post2.md`, `blog/drafts/d1.md`, `notes/idea1.md` +Given files: `blog/post1.md`, `blog/post2.md`, `blog/drafts/d1.md`, +`notes/idea1.md` ``` root/ @@ -69,27 +86,35 @@ At **leaf nodes**, `all` and `any` are computed directly from the files: $$\text{all}(\text{leaf}) = \bigcap_{f \in \text{leaf.files}} \text{fields}(f)$$ $$\text{any}(\text{leaf}) = \bigcup_{f \in \text{leaf.files}} \text{fields}(f)$$ -At **internal nodes** (directories), both sets are the intersection of children's corresponding sets: +At **internal nodes** (directories), both sets are the intersection of +children's corresponding sets: $$\text{all}(\text{dir}) = \bigcap_{c \in \text{children}(\text{dir})} \text{all}(c)$$ $$\text{any}(\text{dir}) = \bigcap_{c \in \text{children}(\text{dir})} \text{any}(c)$$ -Note: `any` at the parent is an **intersection**, not a union. A field is in `dir.any` only if **every** child has at least one file with that field. This means `dir.any` answers: "which fields appear in every branch of this subtree?" +Note: `any` at the parent is an **intersection**, not a union. A field is in +`dir.any` only if **every** child has at least one file with that field. This +means `dir.any` answers: "which fields appear in every branch of this subtree?" ### Glob pattern semantics: `*` vs `**` The two glob depths map to two kinds of evidence: -| Pattern | Scope | Evidence source | -|---------|-------|-----------------| -| `dir/*` | Files directly in `dir/` | Leaf node (observed direct files only) | +| Pattern | Scope | Evidence source | +| -------- | ----------------------------------- | ---------------------------------------- | +| `dir/*` | Files directly in `dir/` | Leaf node (observed direct files only) | | `dir/**` | All files at any depth under `dir/` | Directory node (aggregated full subtree) | -A **leaf node** has observed only the files directly in its directory. It has no information about subdirectories. Its natural scope is `*` — it can only vouch for what it has seen. +A **leaf node** has observed only the files directly in its directory. It has no +information about subdirectories. Its natural scope is `*` — it can only vouch +for what it has seen. -A **directory node**, after bottom-up merge, holds the aggregated picture of its entire subtree. When it confirms a field is in its `any` or `all`, that claim covers everything below it. Its natural scope is `**`. +A **directory node**, after bottom-up merge, holds the aggregated picture of its +entire subtree. When it confirms a field is in its `any` or `all`, that claim +covers everything below it. Its natural scope is `**`. -This is not an exception or special case. It follows from what each node type represents: +This is not an exception or special case. It follows from what each node type +represents: - Leaf = observed direct files → `*` - Directory = aggregated subtree → `**` @@ -97,30 +122,44 @@ This is not an exception or special case. It follows from what each node type re ### The collapse operation -Collapse works bottom-up, processing each directory node. For each field, the directory node's sets determine the action: +Collapse works bottom-up, processing each directory node. For each field, the +directory node's sets determine the action: -| Field in... | Action on `allowed` | Action on `required` | -|---|---|---| -| `dir.all` | Collapse: remove descendants, add `dir` | Collapse: remove descendants, add `dir` | -| `dir.any \ dir.all` | Collapse: remove descendants, add `dir` | No action | -| neither | No action | No action | +| Field in... | Action on `allowed` | Action on `required` | +| ------------------- | --------------------------------------- | --------------------------------------- | +| `dir.all` | Collapse: remove descendants, add `dir` | Collapse: remove descendants, add `dir` | +| `dir.any \ dir.all` | Collapse: remove descendants, add `dir` | No action | +| neither | No action | No action | -"Remove descendants" means: remove all entries whose path starts with `dir`'s path (component-wise, so `blog` does not match `blog-archive`). Then add `dir`'s path. +"Remove descendants" means: remove all entries whose path starts with `dir`'s +path (component-wise, so `blog` does not match `blog-archive`). Then add `dir`'s +path. -When collapse fires, a leaf's `*` contribution is removed and replaced by the directory's `**`. The upgrade is justified because the directory has aggregated evidence from the full subtree. +When collapse fires, a leaf's `*` contribution is removed and replaced by the +directory's `**`. The upgrade is justified because the directory has aggregated +evidence from the full subtree. -When collapse does *not* fire (field not in `dir.any`), the leaf's `*` stays. This is correct: sibling subtrees don't have the field, so `**` would be an unsupported claim. +When collapse does _not_ fire (field not in `dir.any`), the leaf's `*` stays. +This is correct: sibling subtrees don't have the field, so `**` would be an +unsupported claim. ### Initialization -- **`allowed`** is initialized from leaf nodes' `any` sets. Each field in `leaf.any` gets the leaf's directory path added to its allowed set. -- **`required`** is NOT initialized from leaves. It is only populated during collapse, from directory nodes' `all` sets. +- **`allowed`** is initialized from leaf nodes' `any` sets. Each field in + `leaf.any` gets the leaf's directory path added to its allowed set. +- **`required`** is NOT initialized from leaves. It is only populated during + collapse, from directory nodes' `all` sets. -Why not initialize `required` from leaves? Because a leaf's `all` set covers only direct files. A `required` pattern carries recursive implications — requiring a field at `dir/**` means every file under `dir/`, at any depth, must have it. Only the directory-level `all` (which reflects the full subtree intersection) has the evidence to support that claim. +Why not initialize `required` from leaves? Because a leaf's `all` set covers +only direct files. A `required` pattern carries recursive implications — +requiring a field at `dir/**` means every file under `dir/`, at any depth, must +have it. Only the directory-level `all` (which reflects the full subtree +intersection) has the evidence to support that claim. ### Converting paths to globs -After collapse, each field has a set of directory paths for `allowed` and `required`. Conversion to glob strings: +After collapse, each field has a set of directory paths for `allowed` and +`required`. Conversion to glob strings: - Paths from leaf initialization (not collapsed) → `dir/*` (or `*` for root) - Paths from collapse (directory nodes) → `dir/**` (or `**` for root) @@ -156,15 +195,23 @@ root/ all: {title} any: {title} └── {paper1} [leaf] all: {title} any: {title} ``` -**title**: in `root.all` → collapse all the way up → `allowed = ["**"], required = ["**"]` +**title**: in `root.all` → collapse all the way up → +`allowed = ["**"], required = ["**"]` **tags**: + - Leaf init: allowed paths = {`blog/`, `blog/drafts/`, `notes/`} -- `drafts/` collapse: tags in `drafts.all` → collapse allowed + required under `drafts/`. Allowed: leaf `blog/drafts/` stays (it IS `drafts/`). Required: add `blog/drafts/`. -- `blog/` collapse: tags in `blog.any \ blog.all` → collapse allowed only. Remove `blog/` and `blog/drafts/` from allowed, add `blog/`. Required untouched. -- `notes/` collapse: tags in `notes.all` → collapse both. Allowed: `notes/` stays. Required: add `notes/`. +- `drafts/` collapse: tags in `drafts.all` → collapse allowed + required under + `drafts/`. Allowed: leaf `blog/drafts/` stays (it IS `drafts/`). Required: add + `blog/drafts/`. +- `blog/` collapse: tags in `blog.any \ blog.all` → collapse allowed only. + Remove `blog/` and `blog/drafts/` from allowed, add `blog/`. Required + untouched. +- `notes/` collapse: tags in `notes.all` → collapse both. Allowed: `notes/` + stays. Required: add `notes/`. - `root/`: tags not in `root.any` → no action. -- Result: `allowed = ["blog/**", "notes/**"], required = ["blog/drafts/**", "notes/**"]` +- Result: + `allowed = ["blog/**", "notes/**"], required = ["blog/drafts/**", "notes/**"]` ### Example 2: Leaf next to subdirectory (the `*` vs `**` case) @@ -192,14 +239,18 @@ root/ all: {title} any: {title} ``` **deep**: + - Leaf init: allowed paths = {`a/b/`, `a/b/c/d/`} (from leaf `any` sets) -- `d/` collapse: deep in `d.all` → collapse both. Allowed: `a/b/c/d/` stays. Required: add `a/b/c/d/`. +- `d/` collapse: deep in `d.all` → collapse both. Allowed: `a/b/c/d/` stays. + Required: add `a/b/c/d/`. - `c/`: deep not in `c.any` → no action. -- `b/`: deep not in `b.any` (`b.any` = `{title,deep}` $\cap$ `{title}` = `{title}`) → no action. +- `b/`: deep not in `b.any` (`b.any` = `{title,deep}` $\cap$ `{title}` = + `{title}`) → no action. - Leaf `a/b/` keeps its `*` pattern. It was never collapsed. - Result: `allowed = ["a/b/*", "a/b/c/d/**"], required = ["a/b/c/d/**"]` -This is precise: `a/b/*` matches `file4.md` but not `a/b/c/file3.md`. If we had used `a/b/**`, it would incorrectly allow deep in `a/b/c/file3.md`. +This is precise: `a/b/*` matches `file4.md` but not `a/b/c/file3.md`. If we had +used `a/b/**`, it would incorrectly allow deep in `a/b/c/file3.md`. ### Example 3: Files at root alongside subdirectories @@ -213,6 +264,7 @@ x/file5.md → {title} ``` **deep**: + - Leaf init: allowed paths = {`""` (root), `a/b/c/d/`} - `d/` collapse: deep in `d.all` → `a/b/c/d/**`. Required: `a/b/c/d/**`. - No other directory has deep in `any`. @@ -232,13 +284,16 @@ x/file5.md → {title} ``` **deep**: + - Leaf init: allowed paths = {`a/b/e/`, `a/b/c/d/`} - `e/` collapse: deep in `e.all` → `a/b/e/**`. Required: `a/b/e/**`. - `d/` collapse: deep in `d.all` → `a/b/c/d/**`. Required: `a/b/c/d/**`. - No further collapse (deep not in `b.any` because `c.any` = {title}). -- Result: `allowed = ["a/b/c/d/**", "a/b/e/**"], required = ["a/b/c/d/**", "a/b/e/**"]` +- Result: + `allowed = ["a/b/c/d/**", "a/b/e/**"], required = ["a/b/c/d/**", "a/b/e/**"]` -No `*` needed — each occurrence of deep is in its own subdirectory. `**` is justified at both. +No `*` needed — each occurrence of deep is in its own subdirectory. `**` is +justified at both. --- @@ -251,18 +306,21 @@ Three phases: **build**, **merge**, **collapse**. Walk all file paths. For each file: 1. Extract the parent directory path. -2. Ensure the full directory chain exists in the tree (mkdir -p style). Each directory becomes an internal node. +2. Ensure the full directory chain exists in the tree (mkdir -p style). Each + directory becomes an internal node. 3. Get or create the file-set leaf for that directory: - First file in a directory: create the leaf with `all = any = fields`. - Subsequent files: intersect `all`, union `any`. Two maps track NodeIds: + - `dir_map: HashMap` — directory path → directory node - `leaf_map: HashMap` — directory path → file-set leaf node ### Phase 2: Merge (bottom-up) -Traverse the tree bottom-up (via `NodeEdge::End` in post-order). For each internal node (has children): +Traverse the tree bottom-up (via `NodeEdge::End` in post-order). For each +internal node (has children): ``` node.all = intersect(child.all for each child) @@ -275,15 +333,18 @@ Leaf nodes are skipped — they were populated during build. **Initialize:** -- For each leaf node, for each field in `leaf.any`: add the leaf's directory path to `allowed[field]`. Mark these entries as leaf-sourced. +- For each leaf node, for each field in `leaf.any`: add the leaf's directory + path to `allowed[field]`. Mark these entries as leaf-sourced. - `required` starts empty. **Collapse loop** (post-order, directory nodes only): For each field at each directory node: -1. **Field in `node.all`**: collapse both `allowed` and `required` — remove descendant paths, add this node's path (marked as directory-sourced). -2. **Field in `node.any \ node.all`**: collapse `allowed` only — remove descendant paths, add this node's path (marked as directory-sourced). +1. **Field in `node.all`**: collapse both `allowed` and `required` — remove + descendant paths, add this node's path (marked as directory-sourced). +2. **Field in `node.any \ node.all`**: collapse `allowed` only — remove + descendant paths, add this node's path (marked as directory-sourced). 3. **Field not in `node.any`**: skip (descendants keep their entries). **Convert to globs:** @@ -297,27 +358,41 @@ For each field at each directory node: ### Correctness invariants -1. **`required ⊆ allowed`**: Required patterns are only added during collapse when `node.all` has the field, and `node.all ⊆ node.any`, so collapse also adds to allowed. -2. **No false negatives in `allowed`**: Every file that has a field is covered by at least one allowed pattern (initialized from the leaf that contains the file). -3. **No false positives in `required`**: A required pattern at `dir/**` means every file in the subtree has the field (because `node.all` is the intersection of all descendants' `all` sets). -4. **Deterministic output**: `BTreeMap` for fields, sorted glob vectors. Input order does not affect output. +1. **`required ⊆ allowed`**: Required patterns are only added during collapse + when `node.all` has the field, and `node.all ⊆ node.any`, so collapse also + adds to allowed. +2. **No false negatives in `allowed`**: Every file that has a field is covered + by at least one allowed pattern (initialized from the leaf that contains the + file). +3. **No false positives in `required`**: A required pattern at `dir/**` means + every file in the subtree has the field (because `node.all` is the + intersection of all descendants' `all` sets). +4. **Deterministic output**: `BTreeMap` for fields, sorted glob vectors. Input + order does not affect output. ### Conservative direction -The algorithm is conservative in the **permissive** direction for `allowed` — when uncertain, it allows more rather than less. Specifically: +The algorithm is conservative in the **permissive** direction for `allowed` — +when uncertain, it allows more rather than less. Specifically: -- `*` patterns allow any file directly in a directory, even if only some observed files have the field. -- `**` patterns (from collapse) allow any file in the subtree, even at depths not yet observed. +- `*` patterns allow any file directly in a directory, even if only some + observed files have the field. +- `**` patterns (from collapse) allow any file in the subtree, even at depths + not yet observed. -This is intentional: `allowed` is a soft boundary. The user tightens it by hand-editing the config. `required`, by contrast, is strict — only emitted when there is full evidence. +This is intentional: `allowed` is a soft boundary. The user tightens it by +hand-editing the config. `required`, by contrast, is strict — only emitted when +there is full evidence. ### Monotonicity Adding more files to the input can: + - Add new fields to the output. - Expand `allowed` patterns (new locations observed). - Shrink `required` patterns (a file without the field breaks the `all` chain). -- Never produce a contradiction with prior output (patterns only widen or narrow, never flip). +- Never produce a contradiction with prior output (patterns only widen or + narrow, never flip). --- @@ -340,7 +415,8 @@ pub fn infer_field_paths( ### Dependencies -- `indextree` (v4.7) — arena-allocated tree with `NodeId` handles and `traverse()` for post-order walking. +- `indextree` (v4.7) — arena-allocated tree with `NodeId` handles and + `traverse()` for post-order walking. ### Internal structure @@ -349,18 +425,26 @@ pub fn infer_field_paths( - `traverse_and_merge()` — phase 2, mutates arena in-place. - `collapse()` → `BTreeMap` — phase 3, reads arena. - `ensure_dir()` — creates directory chain, returns target NodeId. -- `collapse_paths()` — removes descendants via `Path::starts_with()`, adds ancestor. +- `collapse_paths()` — removes descendants via `Path::starts_with()`, adds + ancestor. - `paths_to_globs()` — converts `PathBuf` set to sorted glob strings. ### Key implementation patterns -**Collect-then-mutate**: `traverse()` borrows the arena immutably. To mutate nodes during post-order traversal, collect `NodeId`s into a `Vec` first, then iterate the vec and mutate. +**Collect-then-mutate**: `traverse()` borrows the arena immutably. To mutate +nodes during post-order traversal, collect `NodeId`s into a `Vec` first, then +iterate the vec and mutate. -**`Path::starts_with()`**: Component-based matching, not string prefix. `"blog"` does not start with `"blog-archive"`. This prevents false collapses on similar-looking paths. +**`Path::starts_with()`**: Component-based matching, not string prefix. `"blog"` +does not start with `"blog-archive"`. This prevents false collapses on +similar-looking paths. ### Implementation status -Fully implemented in `crates/mdvs-schema/src/inference.rs` with `*` vs `**` glob depth tracking via `GlobDepth` enum. 38 inference tests (59 total in crate). Integrated into `mfv init` — scanned files are fed to `infer_field_paths()`, results populate `allowed`/`required` in generated `mfv.toml`. +Fully implemented in `crates/mdvs-schema/src/inference.rs` with `*` vs `**` glob +depth tracking via `GlobDepth` enum. 38 inference tests (59 total in crate). +Integrated into `mfv init` — scanned files are fed to `infer_field_paths()`, +results populate `allowed`/`required` in generated `mfv.toml`. --- diff --git a/docs/spec/archive/30-workflows/init.md b/docs/spec/archive/30-workflows/init.md index 445f044..11f2e7c 100644 --- a/docs/spec/archive/30-workflows/init.md +++ b/docs/spec/archive/30-workflows/init.md @@ -2,26 +2,34 @@ **Status: DRAFT** -**Cross-references:** [Terminology](../01-terminology.md) | [Crate: mfv](../10-crates/mfv/spec.md) | [Crate: mdvs](../10-crates/mdvs/spec.md) | [Crate: mdvs-schema](../10-crates/mdvs-schema/spec.md) | [Workflow: Inference](inference.md) +**Cross-references:** [Terminology](../01-terminology.md) | +[Crate: mfv](../10-crates/mfv/spec.md) | +[Crate: mdvs](../10-crates/mdvs/spec.md) | +[Crate: mdvs-schema](../10-crates/mdvs-schema/spec.md) | +[Workflow: Inference](inference.md) --- ## Overview -The init workflow creates a new field schema and lock file by scanning markdown files, discovering fields, and inferring allowed/required patterns via tree inference. +The init workflow creates a new field schema and lock file by scanning markdown +files, discovering fields, and inferring allowed/required patterns via tree +inference. -Both tools have an init command. `mfv init` creates `mfv.toml` + `mfv.lock`. `mdvs init` subsumes `mfv init` — it does everything `mfv init` does plus model setup, producing `mdvs.toml` + `mdvs.lock`. +Both tools have an init command. `mfv init` creates `mfv.toml` + `mfv.lock`. +`mdvs init` subsumes `mfv init` — it does everything `mfv init` does plus model +setup, producing `mdvs.toml` + `mdvs.lock`. --- ## Actors -| Actor | Role | -|---|---| -| **User** | Invokes init | -| **CLI** | Orchestrates the workflow | -| **Filesystem** | Source of `.md` files | -| **gray_matter** | Frontmatter extraction | +| Actor | Role | +| --------------- | ---------------------------------------------------------------- | +| **User** | Invokes init | +| **CLI** | Orchestrates the workflow | +| **Filesystem** | Source of `.md` files | +| **gray_matter** | Frontmatter extraction | | **mdvs-schema** | Field discovery, type inference, tree inference, TOML generation | --- @@ -34,13 +42,13 @@ mfv init [--dir ] [--glob ] [--config ] [--force] [--dry-ru ### CLI Flags -| Flag | Default | Description | -|---|---|---| -| `--dir ` | `.` | Directory to scan | -| `--glob ` | `**` | File matching glob | -| `--config ` | `mfv.toml` | Output config file path | -| `--force` | off | Overwrite existing config and lock | -| `--dry-run` | off | Print table only, write nothing | +| Flag | Default | Description | +| ------------------ | ---------- | ---------------------------------- | +| `--dir ` | `.` | Directory to scan | +| `--glob ` | `**` | File matching glob | +| `--config ` | `mfv.toml` | Output config file path | +| `--force` | off | Overwrite existing config and lock | +| `--dry-run` | off | Print table only, write nothing | ### Flow @@ -105,13 +113,13 @@ Scanning . ### End States -| State | Condition | Exit Code | -|---|---|---| -| **Success** | Config and lock written | 0 | -| **Dry-run** | Table printed, no files written | 0 | -| **Config exists** | Error: config already exists (suggest `--force`) | 2 | -| **No files found** | Error: no markdown files match the glob | 2 | -| **Dir not found** | Error: specified directory doesn't exist | 2 | +| State | Condition | Exit Code | +| ------------------ | ------------------------------------------------ | --------- | +| **Success** | Config and lock written | 0 | +| **Dry-run** | Table printed, no files written | 0 | +| **Config exists** | Error: config already exists (suggest `--force`) | 2 | +| **No files found** | Error: no markdown files match the glob | 2 | +| **Dir not found** | Error: specified directory doesn't exist | 2 | --- @@ -119,11 +127,15 @@ Scanning . ### `mfv.toml` -Contains `[directory]` section and `[[fields.field]]` entries. Each field has explicit `allowed` and `required` patterns from tree inference. See [Configuration](../40-configuration/frontmatter-toml.md). +Contains `[directory]` section and `[[fields.field]]` entries. Each field has +explicit `allowed` and `required` patterns from tree inference. See +[Configuration](../40-configuration/frontmatter-toml.md). ### `mfv.lock` -Contains `[discovery]` metadata and `[[field]]` entries with per-file observation lists. See [Configuration: Lock File](../40-configuration/frontmatter-toml.md#lock-file-mfvlock). +Contains `[discovery]` metadata and `[[field]]` entries with per-file +observation lists. See +[Configuration: Lock File](../40-configuration/frontmatter-toml.md#lock-file-mfvlock). --- @@ -134,12 +146,13 @@ mdvs init [path] [--model ] [--glob ] [--config ] [--force] [--dry-run] [--include-bare-files] ``` -`mdvs init` subsumes `mfv init`. It performs all the same field discovery and inference steps, then adds model setup. +`mdvs init` subsumes `mfv init`. It performs all the same field discovery and +inference steps, then adds model setup. ### Additional Flags (beyond mfv init) -| Flag | Default | Description | -|---|---|---| +| Flag | Default | Description | +| -------------- | ------------------------------------ | -------------------- | | `--model ` | `minishlab/potion-multilingual-128M` | HuggingFace model ID | ### Flow @@ -198,7 +211,8 @@ sequenceDiagram #### `mdvs.toml` -Contains everything `mfv.toml` has (`[directory]` + `[[fields.field]]` entries) plus search-specific sections: +Contains everything `mfv.toml` has (`[directory]` + `[[fields.field]]` entries) +plus search-specific sections: ```toml [directory] @@ -229,7 +243,8 @@ See [Configuration: mdvs.toml](../40-configuration/mdvs-toml.md). #### `mdvs.lock` -Superset of `mfv.lock`. Contains the same field observations plus per-file content hashes and build metadata: +Superset of `mfv.lock`. Contains the same field observations plus per-file +content hashes and build metadata: ```toml # Auto-generated by mdvs init. Do not edit. @@ -266,19 +281,21 @@ model_revision = "a1b2c3d4e5f6" max_chunk_size = 1000 ``` -The `[build]` section is written at init (with model identity from download) and updated after each `mdvs build`. The `[[file]]` entries are updated by both `mdvs update` and `mdvs build`. +The `[build]` section is written at init (with model identity from download) and +updated after each `mdvs build`. The `[[file]]` entries are updated by both +`mdvs update` and `mdvs build`. --- ## Edge Cases -| Case | Behavior | -|---|---| -| Config already exists | Error with exit 2, suggests `--force`. With `--force`, overwrites both config and lock. | -| All files lack frontmatter | Error: no markdown files found (only files with frontmatter count). | -| Mixed frontmatter formats (YAML/TOML) | `gray_matter` handles both. Type inference works across formats. | -| Single file | Tree inference still works; produces root-level `*` patterns. | -| `--dry-run` with `--config` | Config path is accepted but no files are written. | +| Case | Behavior | +| ------------------------------------- | --------------------------------------------------------------------------------------- | +| Config already exists | Error with exit 2, suggests `--force`. With `--force`, overwrites both config and lock. | +| All files lack frontmatter | Error: no markdown files found (only files with frontmatter count). | +| Mixed frontmatter formats (YAML/TOML) | `gray_matter` handles both. Type inference works across formats. | +| Single file | Tree inference still works; produces root-level `*` patterns. | +| `--dry-run` with `--config` | Config path is accepted but no files are written. | --- @@ -286,7 +303,10 @@ The `[build]` section is written at init (with model identity from download) and - [Crate: mfv](../10-crates/mfv/spec.md) — `mfv init` command implementation - [Crate: mdvs](../10-crates/mdvs/spec.md) — `mdvs init` command implementation -- [Crate: mdvs-schema](../10-crates/mdvs-schema/spec.md) — `discover_fields`, `infer_field_paths` +- [Crate: mdvs-schema](../10-crates/mdvs-schema/spec.md) — `discover_fields`, + `infer_field_paths` - [Workflow: Inference](inference.md) — tree inference algorithm -- [Configuration: Field Schema](../40-configuration/frontmatter-toml.md) — generated file formats -- [Configuration: mdvs.toml](../40-configuration/mdvs-toml.md) — search-specific config sections +- [Configuration: Field Schema](../40-configuration/frontmatter-toml.md) — + generated file formats +- [Configuration: mdvs.toml](../40-configuration/mdvs-toml.md) — search-specific + config sections diff --git a/docs/spec/archive/30-workflows/model-loading.md b/docs/spec/archive/30-workflows/model-loading.md index 5169f91..6ac7a56 100644 --- a/docs/spec/archive/30-workflows/model-loading.md +++ b/docs/spec/archive/30-workflows/model-loading.md @@ -2,13 +2,18 @@ **Status: DRAFT** -**Cross-references:** [Terminology](../01-terminology.md) | [Crate: mdvs](../10-crates/mdvs/spec.md) | [Model Mismatch](model-mismatch.md) | [Configuration: mdvs.toml](../40-configuration/mdvs-toml.md) +**Cross-references:** [Terminology](../01-terminology.md) | +[Crate: mdvs](../10-crates/mdvs/spec.md) | [Model Mismatch](model-mismatch.md) | +[Configuration: mdvs.toml](../40-configuration/mdvs-toml.md) --- ## Overview -mdvs supports two static embedding model formats. Both are structurally identical — a 2D float32 matrix in safetensors plus a HuggingFace `tokenizer.json` — differing only in filenames and tensor keys. A universal loader detects the format and returns a uniform interface. +mdvs supports two static embedding model formats. Both are structurally +identical — a 2D float32 matrix in safetensors plus a HuggingFace +`tokenizer.json` — differing only in filenames and tensor keys. A universal +loader detects the format and returns a uniform interface. --- @@ -23,7 +28,8 @@ model_dir/ └── config.json # model metadata ``` -**Models:** `potion-base-2M` (64-dim), `potion-base-32M`, `potion-retrieval-32M` (512-dim), `potion-multilingual-128M` +**Models:** `potion-base-2M` (64-dim), `potion-base-32M`, `potion-retrieval-32M` +(512-dim), `potion-multilingual-128M` ### Format 2: Sentence Transformers StaticEmbedding @@ -37,18 +43,20 @@ model_dir/ └── config.json # pooling config (ignored) ``` -**Models:** `static-retrieval-mrl-en-v1` (1024-dim, Matryoshka), `static-similarity-mrl-multilingual-v1` (1024-dim, 50 languages) +**Models:** `static-retrieval-mrl-en-v1` (1024-dim, Matryoshka), +`static-similarity-mrl-multilingual-v1` (1024-dim, 50 languages) ### Format Differences -| | Model2Vec | ST StaticEmbedding | -|---|---|---| -| Safetensors filename | `embeddings.safetensors` | `model.safetensors` | -| Tensor key | `"embeddings"` | `"embedding.weight"` | -| Extra config files | `config.json` | `modules.json`, ST config, pooling config | -| Tokenizer | `tokenizer.json` | `tokenizer.json` (identical format) | +| | Model2Vec | ST StaticEmbedding | +| -------------------- | ------------------------ | ----------------------------------------- | +| Safetensors filename | `embeddings.safetensors` | `model.safetensors` | +| Tensor key | `"embeddings"` | `"embedding.weight"` | +| Extra config files | `config.json` | `modules.json`, ST config, pooling config | +| Tokenizer | `tokenizer.json` | `tokenizer.json` (identical format) | -The extra ST config files describe the PyTorch module pipeline, which is irrelevant — mdvs does tokenize → lookup → mean pool directly in Rust. +The extra ST config files describe the PyTorch module pipeline, which is +irrelevant — mdvs does tokenize → lookup → mean pool directly in Rust. --- @@ -73,7 +81,8 @@ flowchart LR 4. **Normalize** — optional L2 normalization 5. **Truncate** — optional Matryoshka dimension reduction (ST models only) -No attention, no transformer forward pass, no GPU. Inference is O(tokens) array lookups + one averaging pass. +No attention, no transformer forward pass, no GPU. Inference is O(tokens) array +lookups + one averaging pass. --- @@ -119,12 +128,14 @@ impl StaticEmbeddingModel { ## Matryoshka Truncation -ST's `static-retrieval-mrl-en-v1` is trained with Matryoshka Representation Learning, meaning embeddings can be truncated to smaller dimensions with minimal quality loss. +ST's `static-retrieval-mrl-en-v1` is trained with Matryoshka Representation +Learning, meaning embeddings can be truncated to smaller dimensions with minimal +quality loss. -| Full dim | Truncated dim | Use case | -|---|---|---| -| 1024 | 512 | Good balance of quality and storage | -| 1024 | 256 | Compact, fast search, slight quality reduction | +| Full dim | Truncated dim | Use case | +| -------- | ------------- | ---------------------------------------------- | +| 1024 | 512 | Good balance of quality and storage | +| 1024 | 256 | Compact, fast search, slight quality reduction | Configured in `mdvs.toml`: @@ -134,7 +145,8 @@ name = "sentence-transformers/static-retrieval-mrl-en-v1" truncate_dim = 256 # optional, Matryoshka truncation ``` -When `truncate_dim` is set, the loader truncates each embedding after mean pooling: +When `truncate_dim` is set, the loader truncates each embedding after mean +pooling: ```rust fn truncate_embedding(embedding: &[f32], target_dim: usize) -> Vec { @@ -142,21 +154,24 @@ fn truncate_embedding(embedding: &[f32], target_dim: usize) -> Vec { } ``` -The `model_dimension` stored in `vault_meta` reflects the truncated dimension, not the model's native dimension. This ensures schema consistency with `FLOAT[N]`. +The `model_dimension` stored in `vault_meta` reflects the truncated dimension, +not the model's native dimension. This ensures schema consistency with +`FLOAT[N]`. --- ## Implementation Phasing -| Phase | What | Details | -|---|---|---| -| v0.1 | Model2Vec-only via `model2vec-rs` crate | Already validated in spike 02 | +| Phase | What | Details | +| ----- | ----------------------------------------- | ---------------------------------------------- | +| v0.1 | Model2Vec-only via `model2vec-rs` crate | Already validated in spike 02 | | v0.3+ | Universal loader replacing `model2vec-rs` | Direct `safetensors` + `tokenizers`, ~50 lines | -| v0.3+ | Matryoshka `truncate_dim` config | ~10 lines | +| v0.3+ | Matryoshka `truncate_dim` config | ~10 lines | ### Why replace `model2vec-rs`? -`model2vec-rs` is a thin wrapper around the same `safetensors` + `tokenizers` crates. Writing the loader directly: +`model2vec-rs` is a thin wrapper around the same `safetensors` + `tokenizers` +crates. Writing the loader directly: - Removes a dependency with uncertain long-term maintenance - Adds ST format support for free (different file/key name) @@ -165,7 +180,8 @@ The `model_dimension` stored in `vault_meta` reflects the truncated dimension, n ### Why not in v0.1? -v0.1 validates that all pieces fit together. `model2vec-rs` works for that. The universal loader is a clean swap for v0.3 when the architecture is stable. +v0.1 validates that all pieces fit together. `model2vec-rs` works for that. The +universal loader is a clean swap for v0.3 when the architecture is stable. --- @@ -173,20 +189,22 @@ v0.1 validates that all pieces fit together. `model2vec-rs` works for that. The Properties relevant to mdvs, validated in spike notebooks: -| Property | Value | Implication | -|---|---|---| -| Context window | **None** — token average, not transformer | No input length limit | +| Property | Value | Implication | +| ------------------ | ------------------------------------------------ | ---------------------------------------------------- | +| Context window | **None** — token average, not transformer | No input length limit | | Embedding dilution | Similarity drops to ~0 at 5:1 noise:signal ratio | Chunking is about semantic quality, not model limits | -| Inference speed | O(tokens) lookups + mean | Effectively instant, no GPU | -| Unicode handling | Tokenizer handles all scripts | No special treatment needed | -| Output | `Vec` of fixed `embedding_dim` | Stored as `FixedSizeList(N)` in Parquet | +| Inference speed | O(tokens) lookups + mean | Effectively instant, no GPU | +| Unicode handling | Tokenizer handles all scripts | No special treatment needed | +| Output | `Vec` of fixed `embedding_dim` | Stored as `FixedSizeList(N)` in Parquet | --- ## Related Documents -- [Terminology](../01-terminology.md) — definitions for Model2Vec, POTION Model, Model Identity +- [Terminology](../01-terminology.md) — definitions for Model2Vec, POTION Model, + Model Identity - [Crate: mdvs](../10-crates/mdvs/spec.md) — `embed` module - [Workflow: Model Mismatch](model-mismatch.md) — identity checks and reindex -- [Configuration: mdvs.toml](../40-configuration/mdvs-toml.md) — `[model]` section +- [Configuration: mdvs.toml](../40-configuration/mdvs-toml.md) — `[model]` + section - [Storage Schema](../20-storage/schema.md) — embedding column, type mappings diff --git a/docs/spec/archive/30-workflows/model-mismatch.md b/docs/spec/archive/30-workflows/model-mismatch.md index 0a4cf3f..c9095ea 100644 --- a/docs/spec/archive/30-workflows/model-mismatch.md +++ b/docs/spec/archive/30-workflows/model-mismatch.md @@ -2,13 +2,18 @@ **Status: DRAFT** -**Cross-references:** [Terminology](../01-terminology.md) | [Crate: mdvs](../10-crates/mdvs/spec.md) | [Storage Schema](../20-storage/schema.md) | [Model Loading](model-loading.md) +**Cross-references:** [Terminology](../01-terminology.md) | +[Crate: mdvs](../10-crates/mdvs/spec.md) | +[Storage Schema](../20-storage/schema.md) | [Model Loading](model-loading.md) --- ## Overview -Embeddings from different models (or different versions of the same model) are incompatible. Mixing them in the same index produces meaningless search results. Every operation that touches embeddings (`index`, `search`, `similar`) checks model identity before proceeding. +Embeddings from different models (or different versions of the same model) are +incompatible. Mixing them in the same index produces meaningless search results. +Every operation that touches embeddings (`index`, `search`, `similar`) checks +model identity before proceeding. --- @@ -16,15 +21,19 @@ Embeddings from different models (or different versions of the same model) are i Three values stored in `vault_meta`: -| Field | Source | Example | -|---|---|---| -| `model_id` | HuggingFace repo ID | `minishlab/potion-multilingual-128M` | -| `model_dimension` | Output vector size | `256` | -| `model_revision` | Git commit SHA of downloaded snapshot | `a1b2c3d4e5f6` | +| Field | Source | Example | +| ----------------- | ------------------------------------- | ------------------------------------ | +| `model_id` | HuggingFace repo ID | `minishlab/potion-multilingual-128M` | +| `model_dimension` | Output vector size | `256` | +| `model_revision` | Git commit SHA of downloaded snapshot | `a1b2c3d4e5f6` | -The revision is resolved from the HuggingFace cache directory structure (`~/.cache/huggingface/hub/models--org--name/snapshots//`). See [Model Loading](model-loading.md) for format details. +The revision is resolved from the HuggingFace cache directory structure +(`~/.cache/huggingface/hub/models--org--name/snapshots//`). See +[Model Loading](model-loading.md) for format details. -**Note:** When Matryoshka truncation is configured, `model_dimension` reflects the truncated dimension (e.g., 256), not the model's native dimension (e.g., 1024). +**Note:** When Matryoshka truncation is configured, `model_dimension` reflects +the truncated dimension (e.g., 256), not the model's native dimension (e.g., +1024). --- @@ -45,7 +54,8 @@ flowchart TD ### Case 1: Dimension Mismatch — Hard Error (always) -The `FLOAT[N]` column would reject vectors of a different size. This is always fatal. +The `FLOAT[N]` column would reject vectors of a different size. This is always +fatal. ``` Error: Dimension mismatch. @@ -57,7 +67,8 @@ Run `mdvs reindex` to rebuild with the new model. ### Case 2: Model ID Mismatch — Hard Error (always) -Different models produce incompatible embedding spaces, even if dimensions happen to match. +Different models produce incompatible embedding spaces, even if dimensions +happen to match. ``` Error: Model mismatch. @@ -72,9 +83,11 @@ Options: ### Case 3: Revision Mismatch — Depends on Operation -Same model ID, different commit SHA. The model weights have been updated on HuggingFace. +Same model ID, different commit SHA. The model weights have been updated on +HuggingFace. -**For `search` and `similar` (read-only):** Warning. Vectors are likely close but not identical. Results may be slightly inconsistent. +**For `search` and `similar` (read-only):** Warning. Vectors are likely close +but not identical. Results may be slightly inconsistent. ``` Warning: Model revision changed. @@ -84,7 +97,8 @@ Warning: Model revision changed. Results may be slightly inconsistent. Run `mdvs reindex` for clean results. ``` -**For `index` (writes new embeddings):** Hard error. We must not mix embeddings from different revisions in the same index. +**For `index` (writes new embeddings):** Hard error. We must not mix embeddings +from different revisions in the same index. ``` Error: Model revision mismatch. @@ -109,12 +123,14 @@ Proceed normally. No output. 1. Load the new model, resolve its identity 2. Update `vault_meta` with new model_id, model_dimension, model_revision -3. If dimension changed: `ALTER TABLE chunks DROP COLUMN embedding; ALTER TABLE chunks ADD COLUMN embedding FLOAT[N_new];` +3. If dimension changed: + `ALTER TABLE chunks DROP COLUMN embedding; ALTER TABLE chunks ADD COLUMN embedding FLOAT[N_new];` 4. `UPDATE chunks SET embedding = NULL` 5. Re-embed all chunks from stored `plain_text` 6. Rebuild HNSW index -Because `plain_text` is stored per-chunk, no filesystem access or re-parsing is needed. For a 5,000-note vault with static embeddings, this takes seconds. +Because `plain_text` is stored per-chunk, no filesystem access or re-parsing is +needed. For a 5,000-note vault with static embeddings, this takes seconds. --- @@ -128,30 +144,35 @@ name = "minishlab/potion-multilingual-128M" revision = "a1b2c3d4e5f6" ``` -When `revision` is set, `model2vec-rs` downloads that exact revision from HuggingFace. This prevents silent model updates from causing revision mismatch warnings. +When `revision` is set, `model2vec-rs` downloads that exact revision from +HuggingFace. This prevents silent model updates from causing revision mismatch +warnings. -When `revision` is omitted, the latest available revision is downloaded. Its SHA is recorded in `vault_meta` at init or reindex time. +When `revision` is omitted, the latest available revision is downloaded. Its SHA +is recorded in `vault_meta` at init or reindex time. --- ## Edge Cases -| Case | Behavior | -|---|---| -| `vault_meta` missing model keys | Error: database may be corrupted or from an older version | -| Model not in HuggingFace cache | `model2vec-rs` downloads it. If no network, error. | -| Pinned revision not available on HuggingFace | Error from `model2vec-rs`: revision not found | -| `--model` flag overrides config | Current model identity uses the flag value. Mismatch rules still apply. | -| `--revision` flag overrides config | Current revision uses the flag value. | +| Case | Behavior | +| -------------------------------------------- | ----------------------------------------------------------------------- | +| `vault_meta` missing model keys | Error: database may be corrupted or from an older version | +| Model not in HuggingFace cache | `model2vec-rs` downloads it. If no network, error. | +| Pinned revision not available on HuggingFace | Error from `model2vec-rs`: revision not found | +| `--model` flag overrides config | Current model identity uses the flag value. Mismatch rules still apply. | +| `--revision` flag overrides config | Current revision uses the flag value. | --- ## Related Documents -- [Workflow: Model Loading](model-loading.md) — format detection, universal loader, inference pipeline +- [Workflow: Model Loading](model-loading.md) — format detection, universal + loader, inference pipeline - [Terminology](../01-terminology.md) — definitions for model identity, reindex - [Crate: mdvs](../10-crates/mdvs/spec.md) — `model` module - [Storage Schema](../20-storage/schema.md) — `vault_meta` keys -- [Configuration: mdvs.toml](../40-configuration/mdvs-toml.md) — model and revision settings +- [Configuration: mdvs.toml](../40-configuration/mdvs-toml.md) — model and + revision settings - [Workflow: Build](build.md) — hard error on revision mismatch during build - [Workflow: Search](search.md) — warning on revision mismatch during search diff --git a/docs/spec/archive/30-workflows/search.md b/docs/spec/archive/30-workflows/search.md index 2978cf5..93231da 100644 --- a/docs/spec/archive/30-workflows/search.md +++ b/docs/spec/archive/30-workflows/search.md @@ -2,25 +2,30 @@ **Status: DRAFT** -**Cross-references:** [Terminology](../01-terminology.md) | [Crate: mdvs](../10-crates/mdvs/spec.md) | [Storage Schema](../20-storage/schema.md) +**Cross-references:** [Terminology](../01-terminology.md) | +[Crate: mdvs](../10-crates/mdvs/spec.md) | +[Storage Schema](../20-storage/schema.md) --- ## Overview -The search workflow embeds a user query, computes cosine distance against chunk embeddings, and returns results ranked at the note level (or chunk level with `--chunks`). Auto-build behavior ensures results reflect the current state of files on disk. +The search workflow embeds a user query, computes cosine distance against chunk +embeddings, and returns results ranked at the note level (or chunk level with +`--chunks`). Auto-build behavior ensures results reflect the current state of +files on disk. --- ## Actors -| Actor | Role | -|---|---| -| **User** | Provides query string and optional filters | -| **CLI** | Orchestrates the search | -| **model2vec-rs** | Embeds the query string | -| **Rust** | Cosine distance computation over Arrow arrays | -| **DataFusion** | SQL JOIN, GROUP BY, WHERE, ORDER BY, LIMIT | +| Actor | Role | +| ---------------- | --------------------------------------------- | +| **User** | Provides query string and optional filters | +| **CLI** | Orchestrates the search | +| **model2vec-rs** | Embeds the query string | +| **Rust** | Cosine distance computation over Arrow arrays | +| **DataFusion** | SQL JOIN, GROUP BY, WHERE, ORDER BY, LIMIT | --- @@ -92,16 +97,17 @@ sequenceDiagram The `on_stale` config key and `--build`/`--no-build` CLI flags interact: -| Config `on_stale` | `--build` | `--no-build` | Result | -|---|---|---|---| -| `auto` | — | — | Build if stale | -| `auto` | — | yes | Skip build | -| `strict` | — | — | Error if stale | -| `strict` | yes | — | Build if stale | -| any | yes | — | Always build | -| any | — | yes | Never build | +| Config `on_stale` | `--build` | `--no-build` | Result | +| ----------------- | --------- | ------------ | -------------- | +| `auto` | — | — | Build if stale | +| `auto` | — | yes | Skip build | +| `strict` | — | — | Error if stale | +| `strict` | yes | — | Build if stale | +| any | yes | — | Always build | +| any | — | yes | Never build | -Staleness is determined by comparing filesystem content hashes against `mdvs.lock` `[[file]]` entries. +Staleness is determined by comparing filesystem content hashes against +`mdvs.lock` `[[file]]` entries. --- @@ -111,8 +117,10 @@ Default mode. Groups chunk results by file and ranks by best chunk match. **Strategy:** -- **Score:** Maximum similarity (minimum cosine distance) across all chunks of a file -- **Snippet:** Plain text of the best-matching chunk (truncated to `snippet_length`) +- **Score:** Maximum similarity (minimum cosine distance) across all chunks of a + file +- **Snippet:** Plain text of the best-matching chunk (truncated to + `snippet_length`) - **Heading:** The heading associated with the best-matching chunk ```sql @@ -130,7 +138,9 @@ ORDER BY distance LIMIT :limit; ``` -The `--where` clause operates on `files` table columns (both schema fields and `metadata` JSON). This gives users the full power of DataFusion SQL for filtering. +The `--where` clause operates on `files` table columns (both schema fields and +`metadata` JSON). This gives users the full power of DataFusion SQL for +filtering. --- @@ -152,7 +162,8 @@ ORDER BY c.distance LIMIT :limit; ``` -Useful for finding specific sections across different files, or when a single long file has multiple relevant sections. +Useful for finding specific sections across different files, or when a single +long file has multiple relevant sections. --- @@ -174,10 +185,9 @@ Useful for finding specific sections across different files, or when a single lo 2 results (8ms search, 1ms embed) ``` -**Line 1:** Rank, filename, `§ heading` (if present), cosine distance. -**Line 2:** Field values (tags, date, etc.). -**Line 3:** Snippet from the best-matching chunk. -**Footer:** Result count, search time, embedding time. +**Line 1:** Rank, filename, `§ heading` (if present), cosine distance. **Line +2:** Field values (tags, date, etc.). **Line 3:** Snippet from the best-matching +chunk. **Footer:** Result count, search time, embedding time. ### JSON @@ -203,7 +213,8 @@ Useful for finding specific sections across different files, or when a single lo } ``` -Field values are included as top-level keys in each result object. The field names depend on the schema. +Field values are included as top-level keys in each result object. The field +names depend on the schema. ### Paths @@ -212,7 +223,8 @@ projects/collabide/crdt-design.md reading/kleppmann-crdt-paper.md ``` -One filename per line. Useful for piping into other tools (`xargs`, `fzf`, editors). +One filename per line. Useful for piping into other tools (`xargs`, `fzf`, +editors). --- @@ -238,21 +250,22 @@ mdvs search "testing" --where "tags @> ['rust'] AND date > '2024-01-01'" ## Edge Cases -| Case | Behavior | -|---|---| -| Empty query string | Error: query must not be empty | -| No results found | Exit code 1, message: "No results found." | -| `--where` with syntax error | DataFusion SQL error, surfaced to user with the invalid clause highlighted | -| `--where` referencing non-existent column | DataFusion error, surfaced to user | -| Artifact not built | Error: ".mdvs/ not found. Run `mdvs build` first." (unless auto-build triggers) | -| Empty artifact (init done, no build yet) | No results (chunks Parquet is empty) | -| Model mismatch | See [Model Mismatch Workflow](model-mismatch.md) | +| Case | Behavior | +| ----------------------------------------- | ------------------------------------------------------------------------------- | +| Empty query string | Error: query must not be empty | +| No results found | Exit code 1, message: "No results found." | +| `--where` with syntax error | DataFusion SQL error, surfaced to user with the invalid clause highlighted | +| `--where` referencing non-existent column | DataFusion error, surfaced to user | +| Artifact not built | Error: ".mdvs/ not found. Run `mdvs build` first." (unless auto-build triggers) | +| Empty artifact (init done, no build yet) | No results (chunks Parquet is empty) | +| Model mismatch | See [Model Mismatch Workflow](model-mismatch.md) | --- ## Related Documents -- [Terminology](../01-terminology.md) — definitions for note-level ranking, embedding, cosine distance +- [Terminology](../01-terminology.md) — definitions for note-level ranking, + embedding, cosine distance - [Crate: mdvs](../10-crates/mdvs/spec.md) — search implementation - [Storage Schema](../20-storage/schema.md) — query patterns - [Workflow: Model Mismatch](model-mismatch.md) — identity check before search diff --git a/docs/spec/archive/40-configuration/frontmatter-toml.md b/docs/spec/archive/40-configuration/frontmatter-toml.md index 5acebf1..b062574 100644 --- a/docs/spec/archive/40-configuration/frontmatter-toml.md +++ b/docs/spec/archive/40-configuration/frontmatter-toml.md @@ -2,19 +2,26 @@ **Status: DRAFT** -**Cross-references:** [Terminology](../01-terminology.md) | [Crate: mdvs-schema](../10-crates/mdvs-schema/spec.md) | [Crate: mfv](../10-crates/mfv/spec.md) +**Cross-references:** [Terminology](../01-terminology.md) | +[Crate: mdvs-schema](../10-crates/mdvs-schema/spec.md) | +[Crate: mfv](../10-crates/mfv/spec.md) --- ## Overview -Field schema file shared between `mfv` and `mdvs`. Defines frontmatter field types and validation rules via `allowed` and `required` glob patterns. Generated by `mfv init` (with inferred patterns from tree inference), then edited by the user. +Field schema file shared between `mfv` and `mdvs`. Defines frontmatter field +types and validation rules via `allowed` and `required` glob patterns. Generated +by `mfv init` (with inferred patterns from tree inference), then edited by the +user. **File naming:** + - `mfv.toml` — used by standalone mfv users - `mdvs.toml` — used by mdvs (also discovered by mfv as fallback) -**Config discovery** (`mfv check`): `--schema` flag → `mfv.toml` → `mdvs.toml` → error. +**Config discovery** (`mfv check`): `--schema` flag → `mfv.toml` → `mdvs.toml` → +error. --- @@ -35,23 +42,24 @@ type = "string" ### `[directory]` Section -| Key | Type | Default | Description | -|---|---|---|---| -| `glob` | string | `"**"` | File glob pattern for discovery and validation | +| Key | Type | Default | Description | +| -------------------- | ------- | ------- | ------------------------------------------------------------------------------- | +| `glob` | string | `"**"` | File glob pattern for discovery and validation | | `include_bare_files` | boolean | `false` | When `true`, files without frontmatter are included in inference and validation | ### `[[fields.field]]` Array of Tables -Each field is defined as an entry in the `[[fields.field]]` array. The `name` key identifies the frontmatter key. +Each field is defined as an entry in the `[[fields.field]]` array. The `name` +key identifies the frontmatter key. -| Key | Type | Default | Description | -|---|---|---|---| -| `name` | string | (required) | Frontmatter key name | -| `type` | string | `"string"` | Field type. One of: `string`, `string[]`, `date`, `boolean`, `integer`, `float`, `enum` | -| `allowed` | string[] | `["**"]` | Glob patterns where this field may appear | -| `required` | string[] | `[]` | Glob patterns where this field must be present | -| `pattern` | string | — | Regex the value must match (string/date types only) | -| `values` | string[] | — | Allowed values (only valid when `type = "enum"`) | +| Key | Type | Default | Description | +| ---------- | -------- | ---------- | --------------------------------------------------------------------------------------- | +| `name` | string | (required) | Frontmatter key name | +| `type` | string | `"string"` | Field type. One of: `string`, `string[]`, `date`, `boolean`, `integer`, `float`, `enum` | +| `allowed` | string[] | `["**"]` | Glob patterns where this field may appear | +| `required` | string[] | `[]` | Glob patterns where this field must be present | +| `pattern` | string | — | Regex the value must match (string/date types only) | +| `values` | string[] | — | Allowed values (only valid when `type = "enum"`) | --- @@ -59,19 +67,21 @@ Each field is defined as an entry in the `[[fields.field]]` array. The `name` ke ### Semantics -Both `allowed` and `required` use glob patterns relative to the vault root. They follow symmetric semantics: +Both `allowed` and `required` use glob patterns relative to the vault root. They +follow symmetric semantics: -| Value | Meaning | -|---|---| -| `[]` (empty) | Nowhere | -| `["**"]` | Everywhere (all files, any depth) | -| `["blog/**"]` | All files under `blog/` recursively | -| `["*"]` | Files in the root directory only (not recursive) | -| `["blog/**", "notes/**"]` | Multiple patterns OR-ed | +| Value | Meaning | +| ------------------------- | ------------------------------------------------ | +| `[]` (empty) | Nowhere | +| `["**"]` | Everywhere (all files, any depth) | +| `["blog/**"]` | All files under `blog/` recursively | +| `["*"]` | Files in the root directory only (not recursive) | +| `["blog/**", "notes/**"]` | Multiple patterns OR-ed | ### Defaults When omitted from the TOML file: + - `allowed` defaults to `["**"]` — a defined field is allowed everywhere - `required` defaults to `[]` — fields are optional by default @@ -86,14 +96,17 @@ type = "string" ### Invariant: `required ⊆ allowed` -A field cannot be required at a path where it isn't allowed. If `required` is non-empty but `allowed` is empty (`[]`), schema parsing fails with a validation error. +A field cannot be required at a path where it isn't allowed. If `required` is +non-empty but `allowed` is empty (`[]`), schema parsing fails with a validation +error. ### `*` vs `**` - `*` matches files in a single directory (shallow) - `**` matches files at any depth (recursive) -Generated by `mfv init` via tree inference: leaf nodes (direct files in a directory) produce `*`, directory nodes (aggregated subtrees) produce `**`. +Generated by `mfv init` via tree inference: leaf nodes (direct files in a +directory) produce `*`, directory nodes (aggregated subtrees) produce `**`. --- @@ -101,19 +114,21 @@ Generated by `mfv init` via tree inference: leaf nodes (direct files in a direct ### Supported Types -| Type | YAML Example | Notes | -|---|---|---| -| `string` | `title: "My Post"` | Default type when omitted | -| `string[]` | `tags: [rust, crdt]` | YAML/TOML sequences | -| `date` | `date: 2025-06-12` | ISO 8601 dates | -| `boolean` | `draft: true` | YAML/TOML booleans | -| `integer` | `priority: 3` | Whole numbers | -| `float` | `score: 0.95` | Floating point | -| `enum` | `status: draft` | Validated against `values` list | +| Type | YAML Example | Notes | +| ---------- | -------------------- | ------------------------------- | +| `string` | `title: "My Post"` | Default type when omitted | +| `string[]` | `tags: [rust, crdt]` | YAML/TOML sequences | +| `date` | `date: 2025-06-12` | ISO 8601 dates | +| `boolean` | `draft: true` | YAML/TOML booleans | +| `integer` | `priority: 3` | Whole numbers | +| `float` | `score: 0.95` | Floating point | +| `enum` | `status: draft` | Validated against `values` list | ### Type Inference -When `type` is omitted, it defaults to `"string"`. During `mfv init`, types are inferred from observed values. See [mdvs-schema: Type Inference](../10-crates/mdvs-schema/spec.md#type-inference). +When `type` is omitted, it defaults to `"string"`. During `mfv init`, types are +inferred from observed values. See +[mdvs-schema: Type Inference](../10-crates/mdvs-schema/spec.md#type-inference). --- @@ -128,7 +143,8 @@ type = "string" allowed = ["papers/**"] ``` -If a file outside `papers/` has a `doi` field, validation reports an error. If `allowed` is omitted (defaults to `["**"]`), the field may appear in any file. +If a file outside `papers/` has a `doi` field, validation reports an error. If +`allowed` is omitted (defaults to `["**"]`), the field may appear in any file. ### `required` — Where a field must appear @@ -139,7 +155,8 @@ type = "string" required = ["**"] ``` -Every file must have a `title` field. If `required` is omitted (defaults to `[]`), the field is optional everywhere. +Every file must have a `title` field. If `required` is omitted (defaults to +`[]`), the field is optional everywhere. Path-scoped requirement: @@ -164,7 +181,8 @@ allowed = ["papers/**"] required = ["papers/**"] ``` -Regex the string value must match. Only valid for `string` and `date` type fields. Uses Rust regex syntax. +Regex the string value must match. Only valid for `string` and `date` type +fields. Uses Rust regex syntax. ### `values` @@ -177,7 +195,8 @@ required = ["blog/**"] allowed = ["blog/**"] ``` -Allowed values for enum fields. Only valid when `type = "enum"`. The value must exactly match one of the listed strings. +Allowed values for enum fields. Only valid when `type = "enum"`. The value must +exactly match one of the listed strings. --- @@ -256,7 +275,8 @@ type = "string" ## Lock File (`mfv.lock`) -Auto-generated by `mfv init`. Captures raw per-file observations (like `Cargo.lock`). Not edited by the user. +Auto-generated by `mfv init`. Captures raw per-file observations (like +`Cargo.lock`). Not edited by the user. ### Format @@ -285,19 +305,19 @@ files = ["blog/a.md"] **`[discovery]`**: Metadata about the scan. -| Key | Type | Description | -|---|---|---| -| `total_files` | integer | Total markdown files matched by the glob | -| `files_with_frontmatter` | integer | Files that had parseable frontmatter | -| `glob` | string | Glob pattern used | -| `generated_at` | string | ISO 8601 timestamp | +| Key | Type | Description | +| ------------------------ | ------- | ---------------------------------------- | +| `total_files` | integer | Total markdown files matched by the glob | +| `files_with_frontmatter` | integer | Files that had parseable frontmatter | +| `glob` | string | Glob pattern used | +| `generated_at` | string | ISO 8601 timestamp | **`[[field]]`**: One entry per discovered field. -| Key | Type | Description | -|---|---|---| -| `name` | string | Field name | -| `type` | string | Inferred type | +| Key | Type | Description | +| ------- | -------- | ---------------------------------------------- | +| `name` | string | Field name | +| `type` | string | Inferred type | | `files` | string[] | Exhaustive list of files containing this field | No derived quantities — count is `files.len()`. @@ -309,22 +329,27 @@ No derived quantities — count is `files.len()`. ### `mfv init` Generates `mfv.toml` with: + - `[directory]` section with the glob pattern -- `[[fields.field]]` entries with inferred types and `allowed`/`required` patterns from tree inference +- `[[fields.field]]` entries with inferred types and `allowed`/`required` + patterns from tree inference - No `pattern` or `values` rules (user adds them manually) Also generates `mfv.lock` with per-file observations. ### Existing File -If the config file already exists when `mfv init` runs, it errors with a message suggesting `--force`. With `--force`, it overwrites both the config and lock files. +If the config file already exists when `mfv init` runs, it errors with a message +suggesting `--force`. With `--force`, it overwrites both the config and lock +files. --- ## Related Documents - [Terminology](../01-terminology.md) — definitions for field, field type -- [Crate: mdvs-schema](../10-crates/mdvs-schema/spec.md) — parsing, type inference, tree inference +- [Crate: mdvs-schema](../10-crates/mdvs-schema/spec.md) — parsing, type + inference, tree inference - [Crate: mfv](../10-crates/mfv/spec.md) — validation execution - [Workflow: Init](../30-workflows/init.md) — init workflow - [Workflow: Inference](../30-workflows/inference.md) — tree inference algorithm diff --git a/docs/spec/archive/40-configuration/mdvs-toml.md b/docs/spec/archive/40-configuration/mdvs-toml.md index d0d4037..df39f6d 100644 --- a/docs/spec/archive/40-configuration/mdvs-toml.md +++ b/docs/spec/archive/40-configuration/mdvs-toml.md @@ -2,21 +2,28 @@ **Status: DRAFT** -**Cross-references:** [Terminology](../01-terminology.md) | [Crate: mdvs](../10-crates/mdvs/spec.md) | [Configuration: Field Schema](frontmatter-toml.md) +**Cross-references:** [Terminology](../01-terminology.md) | +[Crate: mdvs](../10-crates/mdvs/spec.md) | +[Configuration: Field Schema](frontmatter-toml.md) --- ## Overview -`mdvs.toml` is the single config file for `mdvs`. It combines the shared field schema (documented in [frontmatter-toml.md](frontmatter-toml.md)) with search-specific sections documented here. `mfv` discovers `mdvs.toml` as a fallback and silently ignores these search-specific sections. +`mdvs.toml` is the single config file for `mdvs`. It combines the shared field +schema (documented in [frontmatter-toml.md](frontmatter-toml.md)) with +search-specific sections documented here. `mfv` discovers `mdvs.toml` as a +fallback and silently ignores these search-specific sections. -**Location:** `mdvs.toml` in the vault root. Generated by `mdvs init`, editable by the user. +**Location:** `mdvs.toml` in the vault root. Generated by `mdvs init`, editable +by the user. --- ## File Format -The search-specific sections are shown below. These appear alongside the `[directory]` and `[[fields.field]]` sections from the field schema. +The search-specific sections are shown below. These appear alongside the +`[directory]` and `[[fields.field]]` sections from the field schema. ```toml # --- field schema sections (see frontmatter-toml.md) --- @@ -52,55 +59,63 @@ snippet_length = 120 ### `[model]` -| Key | Type | Default | Description | -|---|---|---|---| -| `name` | string | `"minishlab/potion-multilingual-128M"` | HuggingFace model ID. Supports both Model2Vec and ST StaticEmbedding formats. | -| `revision` | string | (latest) | Pin to a specific Git commit SHA. If omitted, the latest revision is downloaded and its SHA recorded in `mdvs.lock`. | -| `truncate_dim` | integer | (none) | Matryoshka dimension truncation. Only effective with models trained with MRL (e.g., `static-retrieval-mrl-en-v1`). Must be ≤ native dimension. | +| Key | Type | Default | Description | +| -------------- | ------- | -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | string | `"minishlab/potion-multilingual-128M"` | HuggingFace model ID. Supports both Model2Vec and ST StaticEmbedding formats. | +| `revision` | string | (latest) | Pin to a specific Git commit SHA. If omitted, the latest revision is downloaded and its SHA recorded in `mdvs.lock`. | +| `truncate_dim` | integer | (none) | Matryoshka dimension truncation. Only effective with models trained with MRL (e.g., `static-retrieval-mrl-en-v1`). Must be ≤ native dimension. | -**Revision pinning:** Setting `revision` prevents silent model updates from causing revision mismatch warnings. Recommended for reproducibility. See [Workflow: Model Mismatch](../30-workflows/model-mismatch.md). +**Revision pinning:** Setting `revision` prevents silent model updates from +causing revision mismatch warnings. Recommended for reproducibility. See +[Workflow: Model Mismatch](../30-workflows/model-mismatch.md). -**Override:** The `--model` flag on `mdvs init` overrides the model for initialization. +**Override:** The `--model` flag on `mdvs init` overrides the model for +initialization. ### `[chunking]` -| Key | Type | Default | Description | -|---|---|---|---| -| `max_chunk_size` | integer | `1000` | Maximum chunk size in characters | +| Key | Type | Default | Description | +| ---------------- | ------- | ------- | -------------------------------- | +| `max_chunk_size` | integer | `1000` | Maximum chunk size in characters | -Character-based sizing is sufficient since Model2Vec handles variable-length input. Token-precise splitting is unnecessary. +Character-based sizing is sufficient since Model2Vec handles variable-length +input. Token-precise splitting is unnecessary. **Override:** `--chunk-size` flag on `mdvs init`. ### `[behavior]` -| Key | Type | Default | Description | -|---|---|---|---| +| Key | Type | Default | Description | +| ---------- | ------ | -------- | -------------------------------- | | `on_stale` | string | `"auto"` | Staleness behavior before search | **Staleness modes:** -| Value | Behavior | -|---|---| -| `auto` | Run incremental build before search. Results are always fresh. Minimal overhead for unchanged vaults (hash comparison only). | -| `strict` | Error if any files have changed since last build. Forces explicit `mdvs build` before search. | +| Value | Behavior | +| -------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `auto` | Run incremental build before search. Results are always fresh. Minimal overhead for unchanged vaults (hash comparison only). | +| `strict` | Error if any files have changed since last build. Forces explicit `mdvs build` before search. | -**CLI overrides:** `--build` forces a build regardless of config, `--no-build` skips it. +**CLI overrides:** `--build` forces a build regardless of config, `--no-build` +skips it. ### `[search]` -| Key | Type | Default | Description | -|---|---|---|---| -| `default_limit` | integer | `10` | Default number of results (overridden by `-n`) | -| `snippet_length` | integer | `120` | Maximum characters for result snippets | +| Key | Type | Default | Description | +| ---------------- | ------- | ------- | ---------------------------------------------- | +| `default_limit` | integer | `10` | Default number of results (overridden by `-n`) | +| `snippet_length` | integer | `120` | Maximum characters for result snippets | --- ## Generation -`mdvs init` generates `mdvs.toml` with both the field schema sections (inferred from scanning) and the search-specific sections (with default values). The user can edit it afterward. +`mdvs init` generates `mdvs.toml` with both the field schema sections (inferred +from scanning) and the search-specific sections (with default values). The user +can edit it afterward. -If `mdvs.toml` already exists when `mdvs init` runs: error. Use `--force` to overwrite. +If `mdvs.toml` already exists when `mdvs init` runs: error. Use `--force` to +overwrite. --- @@ -108,7 +123,10 @@ If `mdvs.toml` already exists when `mdvs init` runs: error. Use `--force` to ove - [Terminology](../01-terminology.md) - [Crate: mdvs](../10-crates/mdvs/spec.md) — reads this configuration -- [Workflow: Model Loading](../30-workflows/model-loading.md) — format support, Matryoshka truncation -- [Workflow: Model Mismatch](../30-workflows/model-mismatch.md) — revision pinning +- [Workflow: Model Loading](../30-workflows/model-loading.md) — format support, + Matryoshka truncation +- [Workflow: Model Mismatch](../30-workflows/model-mismatch.md) — revision + pinning - [Workflow: Search](../30-workflows/search.md) — staleness behavior, defaults -- [Configuration: Field Schema](frontmatter-toml.md) — field schema sections in the same file +- [Configuration: Field Schema](frontmatter-toml.md) — field schema sections in + the same file diff --git a/docs/spec/archive/50-operations/distribution.md b/docs/spec/archive/50-operations/distribution.md index af2d1f7..3724e31 100644 --- a/docs/spec/archive/50-operations/distribution.md +++ b/docs/spec/archive/50-operations/distribution.md @@ -8,7 +8,9 @@ ## Overview -Two binaries ship independently: `mdvs` (~20MB, full search) and `mfv` (~2MB, standalone validator). Both are single statically-linked Rust executables with no shared library dependencies. +Two binaries ship independently: `mdvs` (~20MB, full search) and `mfv` (~2MB, +standalone validator). Both are single statically-linked Rust executables with +no shared library dependencies. --- @@ -16,15 +18,15 @@ Two binaries ship independently: `mdvs` (~20MB, full search) and `mfv` (~2MB, st Everything compiled from Rust crates is baked in at build time: -| Component | Included via | -|---|---| -| SQL query engine | `datafusion` crate | -| Parquet I/O | `parquet` + `arrow` crates | -| Model2Vec inference | `model2vec-rs` crate | -| Frontmatter parser | `gray_matter` crate | -| Markdown chunker | `text-splitter` crate | -| Markdown parser | `pulldown-cmark` crate | -| CLI framework | `clap` crate | +| Component | Included via | +| ------------------- | -------------------------- | +| SQL query engine | `datafusion` crate | +| Parquet I/O | `parquet` + `arrow` crates | +| Model2Vec inference | `model2vec-rs` crate | +| Frontmatter parser | `gray_matter` crate | +| Markdown chunker | `text-splitter` crate | +| Markdown parser | `pulldown-cmark` crate | +| CLI framework | `clap` crate | No shared libraries, no runtime interpreters, no system dependencies. @@ -34,11 +36,12 @@ No shared libraries, no runtime interpreters, no system dependencies. `mdvs init` requires network access for one thing: -| Download | Size | Cached At | Required For | -|---|---|---|---| +| Download | Size | Cached At | Required For | +| ----------------------- | --------------- | --------------------------- | ------------------- | | Embedding model weights | ~30MB (default) | `~/.cache/huggingface/hub/` | Embedding inference | -After `init` completes, all subsequent operations (`build`, `search`, etc.) are fully offline. +After `init` completes, all subsequent operations (`build`, `search`, etc.) are +fully offline. A progress bar (via `indicatif`) is shown during model download. @@ -48,13 +51,13 @@ A progress bar (via `indicatif`) is shown during model download. ## Distribution Channels -| Channel | Command | Installs | -|---|---|---| -| **crates.io** | `cargo install mdvs` | Full search tool (~20MB) | -| **crates.io** | `cargo install mfv` | Standalone validator (~2MB) | -| **GitHub Releases** | Download pre-built binary | Both binaries per release | -| **Homebrew tap** | `brew install /tap/mdvs` | Full search tool | -| **Homebrew tap** | `brew install /tap/mfv` | Standalone validator | +| Channel | Command | Installs | +| ------------------- | ------------------------------ | --------------------------- | +| **crates.io** | `cargo install mdvs` | Full search tool (~20MB) | +| **crates.io** | `cargo install mfv` | Standalone validator (~2MB) | +| **GitHub Releases** | Download pre-built binary | Both binaries per release | +| **Homebrew tap** | `brew install /tap/mdvs` | Full search tool | +| **Homebrew tap** | `brew install /tap/mfv` | Standalone validator | --- @@ -64,12 +67,12 @@ Primary distribution path. Built in CI via `cargo-dist` or cross-compilation. ### Target Platforms -| Target | OS | Arch | -|---|---|---| -| `x86_64-unknown-linux-gnu` | Linux | x86_64 | -| `aarch64-unknown-linux-gnu` | Linux | ARM64 | -| `x86_64-apple-darwin` | macOS | Intel | -| `aarch64-apple-darwin` | macOS | Apple Silicon | +| Target | OS | Arch | +| --------------------------- | ----- | ------------- | +| `x86_64-unknown-linux-gnu` | Linux | x86_64 | +| `aarch64-unknown-linux-gnu` | Linux | ARM64 | +| `x86_64-apple-darwin` | macOS | Intel | +| `aarch64-apple-darwin` | macOS | Apple Silicon | Each release is a single compressed binary — download, extract, put in PATH. @@ -77,13 +80,13 @@ Each release is a single compressed binary — download, extract, put in PATH. ## Dependency Comparison -| Tool | Install requires | Runtime requires | -|---|---|---| -| **mdvs** | Download one binary | First-run network for model download | -| **mfv** | Download one binary | Nothing | -| qmd | Node.js/Bun + npm | Ollama running | -| obsidian-note-taking-assistant | Python + pip/uv | Python runtime | -| mdrag | Rust toolchain or binary | Ollama running | +| Tool | Install requires | Runtime requires | +| ------------------------------ | ------------------------ | ------------------------------------ | +| **mdvs** | Download one binary | First-run network for model download | +| **mfv** | Download one binary | Nothing | +| qmd | Node.js/Bun + npm | Ollama running | +| obsidian-note-taking-assistant | Python + pip/uv | Python runtime | +| mdrag | Rust toolchain or binary | Ollama running | --- diff --git a/docs/spec/archive/crate-plan.md b/docs/spec/archive/crate-plan.md index 8234703..16f5bdd 100644 --- a/docs/spec/archive/crate-plan.md +++ b/docs/spec/archive/crate-plan.md @@ -2,7 +2,8 @@ ## Overview -Single crate, single binary. All prototypes from `scripts/` are validated and ready to become real modules. +Single crate, single binary. All prototypes from `scripts/` are validated and +ready to become real modules. Pipeline: **discover** → **schema** → **index** → **search** @@ -83,7 +84,8 @@ uuid = { version = "1", features = ["v4"] } chrono = "0.4" ``` -**No explicit `arrow` or `parquet` deps** — use `datafusion::arrow` and `datafusion::parquet` re-exports to avoid version mismatch. +**No explicit `arrow` or `parquet` deps** — use `datafusion::arrow` and +`datafusion::parquet` re-exports to avoid version mismatch. --- @@ -467,6 +469,7 @@ pub struct SearchResult { Each command is a thin function that wires the library modules together. **`cmd/init.rs`** — The big one: + 1. `ScannedFiles::scan(root, glob, include_bare_files)` 2. `InferredSchema::infer(&scanned)` 3. `MdvsToml::from_inferred(&schema, &config)` → write `mdvs.toml` @@ -477,6 +480,7 @@ Each command is a thin function that wires the library modules together. 8. `build_files_batch` + `build_chunks_batch` → write Parquet to `.mdvs/` **`cmd/build.rs`** — Incremental rebuild: + 1. Read `mdvs.lock` for content hashes 2. `ScannedFiles::scan` for current state 3. Diff: new/modified/deleted files @@ -485,6 +489,7 @@ Each command is a thin function that wires the library modules together. 6. Update lock with new hashes **`cmd/search.rs`**: + 1. Read `.mdvs/files.parquet` + `.mdvs/chunks.parquet` 2. `Embedder::load` + embed query text 3. `CosineSimilarityUDF::new(query_embedding)` @@ -493,20 +498,24 @@ Each command is a thin function that wires the library modules together. 6. Format and print results **`cmd/check.rs`**: + 1. Read schema from `mdvs.toml` 2. `ScannedFiles::scan` 3. Validate each file's frontmatter against field defs 4. Report diagnostics **`cmd/update.rs`**: + 1. Re-scan directory 2. Re-infer (or just update lock with current observations) 3. Rewrite `mdvs.lock` **`cmd/clean.rs`**: + 1. `rm -rf .mdvs/` **`cmd/info.rs`**: + 1. Read `mdvs.toml` + `mdvs.lock` + `.mdvs/` Parquet metadata 2. Print: model, file count, chunk count, field list, staleness @@ -546,49 +555,60 @@ async fn main() { ## Implementation Order -Each step produces a compiling, testable crate. Tests are written alongside each module. +Each step produces a compiling, testable crate. Tests are written alongside each +module. ### Step 1: Skeleton + - `Cargo.toml` with all deps - Empty module stubs (`mod.rs` files with `pub mod` declarations) - `main.rs` with clap parsing (commands exist but print "not implemented") - `cargo build` passes ### Step 2: `discover/field_type.rs` + - `FieldType` enum, `From<&Value>`, `widen`, `Into` - Unit tests (from test_widening.rs + test_arrow.rs) ### Step 3: `discover/scan.rs` + - `ScannedFile`, `ScannedFiles::scan` - Unit tests (from test_scan.rs, uses tempdir) ### Step 4: `discover/infer.rs` + - `DirectoryTree`, `GlobMap`, `InferredSchema` - Unit tests (from test_inference.rs) ### Step 5: `schema/` + - `shared.rs`: `FieldTypeSerde`, `TomlConfig` - `config.rs`: `MdvsToml` - `lock.rs`: `MdvsLock` - Unit tests (from test_toml.rs) ### Step 6: `index/chunk.rs` + - `Chunks`, `extract_plain_text`, `strip_wikilinks` - Unit tests (from test_chunk.rs) ### Step 7: `index/embed.rs` + - `ModelConfig`, `Embedder`, `resolve_revision` - Integration tests (requires model download, from test_embed.rs) ### Step 8: `index/storage.rs` + - Arrow builders + Parquet I/O - Unit tests (from test_parquet.rs, uses tempdir) ### Step 9: `search.rs` + - `CosineSimilarityUDF` - Unit tests (from test_search.rs, uses tempdir + DataFusion) ### Step 10: `cmd/` — wire it all together + - `init` first (exercises the full pipeline) - Then `build`, `search`, `check`, `update`, `clean`, `info` - Integration tests against real directories @@ -597,9 +617,11 @@ Each step produces a compiling, testable crate. Tests are written alongside each ## DataFusion Gotchas (reference) -- **Always use `datafusion::arrow` and `datafusion::parquet` re-exports** — never add separate arrow/parquet deps +- **Always use `datafusion::arrow` and `datafusion::parquet` re-exports** — + never add separate arrow/parquet deps - **Parquet strings → `Utf8View` / `StringViewArray`** — not `StringArray` -- **`ScalarUDFImpl` requires `DynEq + DynHash`** — manual `PartialEq`/`Eq`/`Hash` impls needed +- **`ScalarUDFImpl` requires `DynEq + DynHash`** — manual + `PartialEq`/`Eq`/`Hash` impls needed - **`register_parquet` takes `&str`** path, not `&Path` - **Struct field access in SQL**: `f.data['field_name']` @@ -614,6 +636,7 @@ Each step produces a compiling, testable crate. Tests are written alongside each ``` Config files live at repo root: + ``` mdvs.toml # boundaries (user edits this) mdvs.lock # observed state (auto-generated) diff --git a/docs/spec/archive/decisions.md b/docs/spec/archive/decisions.md index 5a4b5fa..375fb53 100644 --- a/docs/spec/archive/decisions.md +++ b/docs/spec/archive/decisions.md @@ -1,8 +1,8 @@ # Design Decisions -Brainstorming notes and confirmed decisions, organized by topic. -Decisions here are **confirmed but not yet spec'd or implemented**. -Once a decision is implemented and reflected in the specs, remove it from this file. +Brainstorming notes and confirmed decisions, organized by topic. Decisions here +are **confirmed but not yet spec'd or implemented**. Once a decision is +implemented and reflected in the specs, remove it from this file. --- @@ -10,19 +10,20 @@ Once a decision is implemented and reflected in the specs, remove it from this f ### The three artifacts (Cargo analogy) -| mdvs | Cargo | Purpose | Git | -|------|-------|---------|-----| -| `mdvs.toml` | `Cargo.toml` | User-declared intent: schema, model, config | committed | -| `mdvs.lock` | `Cargo.lock` | Resolved state: exact model revision, file hashes, inferred fields | committed | -| `.mdvs/` | `target/` | Build artifacts (parquet files), regenerable from lock + source | gitignored | +| mdvs | Cargo | Purpose | Git | +| ----------- | ------------ | ------------------------------------------------------------------ | ---------- | +| `mdvs.toml` | `Cargo.toml` | User-declared intent: schema, model, config | committed | +| `mdvs.lock` | `Cargo.lock` | Resolved state: exact model revision, file hashes, inferred fields | committed | +| `.mdvs/` | `target/` | Build artifacts (parquet files), regenerable from lock + source | gitignored | -**Why the lock exists separately from the parquets:** -The lock is the resolved inputs, like `Cargo.lock` pinning exact dependency versions. -The parquets are derived build outputs. You can clone a repo, see the lock, and know -exactly what should be indexed — without the parquet files. The lock is human-readable, +**Why the lock exists separately from the parquets:** The lock is the resolved +inputs, like `Cargo.lock` pinning exact dependency versions. The parquets are +derived build outputs. You can clone a repo, see the lock, and know exactly what +should be indexed — without the parquet files. The lock is human-readable, git-diffable, and provides staleness detection without reading binary parquet. The information could technically live in parquet metadata, but: + - Lock is cheap to parse (TOML) vs parquet - Lock is diffable in PRs ("these files changed, this field was added") - Lock exists before build (after init/update, before parquets are generated) @@ -31,25 +32,26 @@ The information could technically live in parquet metadata, but: The tool has two independent value propositions: -1. **Validation layer** — frontmatter schema enforcement. Needs only `mdvs.toml` + - files on disk. No model, no embeddings, no parquets. -2. **Search layer** — semantic search. Needs model + parquets on top of the validation - layer. +1. **Validation layer** — frontmatter schema enforcement. Needs only + `mdvs.toml` + files on disk. No model, no embeddings, no parquets. +2. **Search layer** — semantic search. Needs model + parquets on top of the + validation layer. Commands split cleanly between layers: -| Layer | Commands | -|-------|----------| +| Layer | Commands | +| ---------- | ------------------------- | | Validation | `init`, `update`, `check` | -| Search | `build`, `search` | -| Utility | `clean`, `info` | +| Search | `build`, `search` | +| Utility | `clean`, `info` | -The search layer depends on the validation layer (build needs toml + lock), but the -validation layer stands on its own. +The search layer depends on the validation layer (build needs toml + lock), but +the validation layer stands on its own. ### `init` — first-time setup (infer + update + build) The "set up everything from scratch" command. Does the full pipeline: + 1. Scan files 2. Infer schema — discover field names, types, allowed/required patterns 3. Download model @@ -60,7 +62,8 @@ The "set up everything from scratch" command. Does the full pipeline: `init` is an `update` with inference on top, followed by a `build`. Flag: `--auto-build` (default true) — controls whether init triggers a build at -the end. Written to `mdvs.toml` as `auto_build` so update inherits the preference. +the end. Written to `mdvs.toml` as `auto_build` so update inherits the +preference. ### `update` — re-scan, validate, refresh lock @@ -69,16 +72,19 @@ Like `cargo update` — re-scan files, validate against schema, update the lock. Requires `mdvs.toml` to exist (reads config from it). **Steps:** + 1. Scan files 2. **Pre-check** — validate all files against toml schema (types + paths) 3. Based on `on_error` behavior: - - `fail` (default): validation errors block the update. User must fix files first. - - `skip`: non-conforming files are excluded from the lock with a warning. Only - clean files enter the lock (and therefore the index). + - `fail` (default): validation errors block the update. User must fix files + first. + - `skip`: non-conforming files are excluded from the lock with a warning. + Only clean files enter the lock (and therefore the index). 4. Update `mdvs.lock` with current file hashes and field metadata 5. If `auto_build` is true: trigger `build` **Flags:** + - `--build=true|false` — override `auto_build` from toml (always wins) - `--on-error=fail|skip` — override `on_error` from toml (always wins) - `--infer=new|all|none` — how to handle new/changed fields: @@ -88,6 +94,7 @@ Requires `mdvs.toml` to exist (reads config from it). required nowhere **Toml config:** + ```toml [config] auto_build = true # update triggers build by default @@ -96,19 +103,21 @@ on_error = "fail" # validation errors block the update ### `check` — validate frontmatter against schema -Validate files on disk against the declared schema in `mdvs.toml`. Independent from -update — just scans files and reports violations. No side effects (doesn't modify -lock or parquets). +Validate files on disk against the declared schema in `mdvs.toml`. Independent +from update — just scans files and reports violations. No side effects (doesn't +modify lock or parquets). -This is the same validation that `update` runs as a pre-check, but as a standalone -command for CI or manual inspection. +This is the same validation that `update` runs as a pre-check, but as a +standalone command for CI or manual inspection. **Open questions:** -- What violations to report? Missing required fields, type mismatches, disallowed fields, - unknown/undeclared fields? -- Output format: `file:field: message` lines? Exit 0 if clean, exit 1 if violations? -- Type matching leniency: does Float accept integer values? (Probably yes, since widening - means some files had int and some had float.) + +- What violations to report? Missing required fields, type mismatches, + disallowed fields, unknown/undeclared fields? +- Output format: `file:field: message` lines? Exit 0 if clean, exit 1 if + violations? +- Type matching leniency: does Float accept integer values? (Probably yes, since + widening means some files had int and some had float.) ### `build` — rebuild the search index (expensive) @@ -116,17 +125,18 @@ Read toml, scan files, chunk, embed, write parquets, update lock hashes. This is where the model loads and the real work happens. Use cases: + - After manually editing `mdvs.toml` (changing field types, model, etc.) - After `update --build=false` to rebuild with refreshed lock - Force rebuild ### `search` — query the index -Load model, embed query, run note-level SQL (MAX chunk similarity grouped by file), -print `score filename` to stdout. +Load model, embed query, run note-level SQL (MAX chunk similarity grouped by +file), print `score filename` to stdout. -Flags: `--limit` (default 10), `--path`, `--where` (SQL WHERE clause). -Revision mismatch is a warning (not error). +Flags: `--limit` (default 10), `--path`, `--where` (SQL WHERE clause). Revision +mismatch is a warning (not error). ### `clean` — remove `.mdvs/` directory @@ -134,8 +144,8 @@ Delete build artifacts. Like `cargo clean`. ### `info` — show index status -Display model name/revision, file count, chunk count, staleness (files changed since -last build). +Display model name/revision, file count, chunk count, staleness (files changed +since last build). ### Typical workflows @@ -164,6 +174,7 @@ mdvs update --on-error=skip # warn about bad files, exclude them, build ### Future validation features (post-v0.3) Additional optional constraints on `[[fields.field]]`: + - `min` / `max` for integers and floats (bounds) - `after` / `before` for dates (date ranges) - `values` for string enums (already spec'd) @@ -177,6 +188,7 @@ Additional optional constraints on `[[fields.field]]`: ### `search` vs `query` commands Two separate commands, explicit intent: + - `mdvs search "query text"` — always involves vector similarity - `mdvs query "SELECT ..."` — pure SQL on metadata, no vectors @@ -184,12 +196,12 @@ No auto-detection or magic parsing. ### SQL flags on `search` -| Flag | Maps to | Example | -|------|---------|---------| -| `--where` | WHERE | `--where "tags LIKE '%rust%'"` | -| `--select` | SELECT (extra columns) | `--select "tags, category"` | -| `--order` | ORDER BY (secondary sort) | `--order "date DESC"` | -| `--limit` | LIMIT | `--limit 20` | +| Flag | Maps to | Example | +| ---------- | ------------------------- | ------------------------------ | +| `--where` | WHERE | `--where "tags LIKE '%rust%'"` | +| `--select` | SELECT (extra columns) | `--select "tags, category"` | +| `--order` | ORDER BY (secondary sort) | `--order "date DESC"` | +| `--limit` | LIMIT | `--limit 20` | **`--where` filters BEFORE vector ranking** — filter first, then compute embeddings/distances only on surviving rows. Cheaper and more intuitive. @@ -199,53 +211,61 @@ No `--group-by` for now. ### Storage: `.mdvs/` directory with two Parquet files The artifact is the `.mdvs/` directory (like `target/` in cargo). Contains: -- `files.parquet` — one row per file: `file_id` (UUID), `filename`, `frontmatter` (JSON), `content_hash`, `built_at` -- `chunks.parquet` — one row per chunk: `chunk_id` (UUID), `file_id` (FK), `chunk_index`, `start_line`, `end_line`, `embedding` + +- `files.parquet` — one row per file: `file_id` (UUID), `filename`, + `frontmatter` (JSON), `content_hash`, `built_at` +- `chunks.parquet` — one row per chunk: `chunk_id` (UUID), `file_id` (FK), + `chunk_index`, `start_line`, `end_line`, `embedding` No dynamic field columns — all frontmatter lives in a single JSON column. Simpler schema, no rebuild when field config changes. -No bundling/compression of multiple Parquets together. Parquet already compresses -well internally (zstd per column chunk). Any archive format would kill random access -and memory-mapping, defeating columnar scanning. +No bundling/compression of multiple Parquets together. Parquet already +compresses well internally (zstd per column chunk). Any archive format would +kill random access and memory-mapping, defeating columnar scanning. No raw markdown text stored in either file — keep it lightweight. Search results -show file path + score. User opens the original file for content. `--snippets` flag -reads chunk text from file using line offsets. +show file path + score. User opens the original file for content. `--snippets` +flag reads chunk text from file using line offsets. Model change requires re-reading all files from disk (no cached plain_text). ### Chunk hashing for incremental re-embedding (deferred to v0.4+) Each chunk gets a content hash. On rebuild: + 1. Re-chunk the file (fast, pure text processing) 2. Hash each chunk 3. Compare against stored chunk hashes in `chunks.parquet` 4. Only embed chunks with new/changed hashes 5. Reuse existing embeddings for unchanged chunks -Benefit: adding a paragraph to a long document re-embeds only 1-2 chunks instead of all. -Requires deterministic chunking — `text-splitter` MarkdownSplitter splits on structural -boundaries (headers, paragraphs), so unchanged sections produce identical chunks even -with mid-file edits. +Benefit: adding a paragraph to a long document re-embeds only 1-2 chunks instead +of all. Requires deterministic chunking — `text-splitter` MarkdownSplitter +splits on structural boundaries (headers, paragraphs), so unchanged sections +produce identical chunks even with mid-file edits. -v0.3: file-level hashes only (in `mdvs.lock`). File changed → full re-chunk + re-embed. +v0.3: file-level hashes only (in `mdvs.lock`). File changed → full re-chunk + +re-embed. ### `describe` command (post-v0.3) A command that, given a subpath, shows the shape of the data: -- For each meaningful subpath (only those carrying information, not redundant subpaths): - which fields are allowed and required -- For each field: useful characteristics for query planning — upper/lower bounds, - statistical metrics (mean, variance, etc.) -- Distinction between validation boundaries (configured constraints like "no values < 0") - and observed boundaries (actual data extremes like "smallest value is 1") -- Only meaningful subpaths shown (same logic as inference.rs tree — if field is required - in `folder/**`, don't repeat it for `folder/subfolder/**`) + +- For each meaningful subpath (only those carrying information, not redundant + subpaths): which fields are allowed and required +- For each field: useful characteristics for query planning — upper/lower + bounds, statistical metrics (mean, variance, etc.) +- Distinction between validation boundaries (configured constraints like "no + values < 0") and observed boundaries (actual data extremes like "smallest + value is 1") +- Only meaningful subpaths shown (same logic as inference.rs tree — if field is + required in `folder/**`, don't repeat it for `folder/subfolder/**`) Split between mfv and mdvs: + - mfv: schema view (allowed/required per subpath, configured constraints) - mdvs: adds data statistics on top (observed ranges, means, etc.) -Requires setters/getters for validation boundaries per field (min/max, after/before, etc.) -— ties into "Future validation features" above. +Requires setters/getters for validation boundaries per field (min/max, +after/before, etc.) — ties into "Future validation features" above. diff --git a/docs/spec/archive/v03-config-design.md b/docs/spec/archive/v03-config-design.md index e9176e9..714a84d 100644 --- a/docs/spec/archive/v03-config-design.md +++ b/docs/spec/archive/v03-config-design.md @@ -5,6 +5,7 @@ Captured from discussion on 2026-02-25. ## mdvs.toml Structure Order when auto-generated by `mdvs init`: + 1. `[directory]` 2. `[model]` 3. `[chunking]` @@ -15,7 +16,9 @@ Order when auto-generated by `mdvs init`: User can reorder freely after generation. ### [directory] + Same as mfv.toml, shared schema: + ```toml [directory] glob = "**" # path scope @@ -24,6 +27,7 @@ frontmatter_format = "both" # "both" | "yaml" | "toml" ``` ### [model] + ```toml [model] provider = "huggingface" # fixed in v0.3, error on anything else @@ -31,20 +35,27 @@ id = "minishlab/potion-base-8M" revision = "abc123def456" # always resolved and written by init ``` -- `provider`: forward-compatible field. v0.3 only supports `"huggingface"`. Future: `"ollama"`, `"azure"`, `"gcp"`. -- `revision`: `init` downloads/resolves the model and writes the resolved revision. If user removes it, treat as "any revision is fine" but still record actual revision in lock. -- No `dimension` field: dimension is a model property, discovered at load time. Not user-configurable. +- `provider`: forward-compatible field. v0.3 only supports `"huggingface"`. + Future: `"ollama"`, `"azure"`, `"gcp"`. +- `revision`: `init` downloads/resolves the model and writes the resolved + revision. If user removes it, treat as "any revision is fine" but still record + actual revision in lock. +- No `dimension` field: dimension is a model property, discovered at load time. + Not user-configurable. ### [chunking] + ```toml [chunking] max_chunk_size = 1000 # characters (not bytes) chunk_overlap = 0 # characters of overlap between chunks ``` -- `chunk_overlap`: new field, default 0. `text-splitter` supports overlap natively. +- `chunk_overlap`: new field, default 0. `text-splitter` supports overlap + natively. ### [behavior] + ```toml [behavior] on_stale = "auto" # "auto" | "strict" @@ -54,6 +65,7 @@ on_stale = "auto" # "auto" | "strict" - `strict`: error if stale, require explicit build ### [search] + ```toml [search] default_limit = 10 @@ -63,13 +75,15 @@ default_limit = 10 - `--snippets` CLI flag to show chunk text (full text, no truncation in v0.3). ### [[fields.field]] + Same as mfv.toml. No changes for mdvs. --- ## mdvs.lock Structure -**Core principle**: lock is a superset of the TOML config. Every config value is mirrored so that config changes can be detected as staleness. +**Core principle**: lock is a superset of the TOML config. Every config value is +mirrored so that config changes can be detected as staleness. ```toml # Auto-generated by mdvs. Do not edit. @@ -122,13 +136,18 @@ last_built_at = "2026-02-25T10:05:00Z" ``` ### What's NOT in the lock -- `dimension`: derived from model at load time. Arrow schema encodes it in FixedSizeList. + +- `dimension`: derived from model at load time. Arrow schema encodes it in + FixedSizeList. - `snippet_length`: not in v0.3. -- Field definitions (allowed/required/pattern/values): those are boundaries (TOML), not observed state (lock). +- Field definitions (allowed/required/pattern/values): those are boundaries + (TOML), not observed state (lock). - Chunk hashes: deferred to v0.4+. ### Staleness detection + Compare lock vs current state: + - Config sections changed? → lock is stale, rebuild needed - File hashes changed? → those files need re-processing - Files added/removed? → incremental update @@ -138,10 +157,12 @@ Compare lock vs current state: ## mfv.lock Gap Current `mfv.lock` only mirrors `glob` from config. Needs update to also mirror: + - `include_bare_files` - `frontmatter_format` -This makes `mfv diff` able to detect config changes (e.g., user changed frontmatter_format from "both" to "yaml"). +This makes `mfv diff` able to detect config changes (e.g., user changed +frontmatter_format from "both" to "yaml"). --- @@ -158,28 +179,31 @@ This makes `mfv diff` able to detect config changes (e.g., user changed frontmat ### `files.parquet` -| Column | Arrow Type | Description | -|---|---|---| -| `file_id` | `Utf8` | UUID v4 (primary key) | -| `filename` | `Utf8` | Relative path from vault root | -| `frontmatter` | `Utf8` | Full frontmatter as JSON string | -| `content_hash` | `Utf8` | xxh3 hash of file content | -| `built_at` | `Timestamp(Microsecond, None)` | When last processed | +| Column | Arrow Type | Description | +| -------------- | ------------------------------ | ------------------------------- | +| `file_id` | `Utf8` | UUID v4 (primary key) | +| `filename` | `Utf8` | Relative path from vault root | +| `frontmatter` | `Utf8` | Full frontmatter as JSON string | +| `content_hash` | `Utf8` | xxh3 hash of file content | +| `built_at` | `Timestamp(Microsecond, None)` | When last processed | -No dynamic field columns — all frontmatter in a single JSON column. Simpler schema, no rebuild when field config changes. +No dynamic field columns — all frontmatter in a single JSON column. Simpler +schema, no rebuild when field config changes. ### `chunks.parquet` -| Column | Arrow Type | Description | -|---|---|---| -| `chunk_id` | `Utf8` | UUID v4 | -| `file_id` | `Utf8` | FK to files.parquet | -| `chunk_index` | `Int32` | 0-based position within file | -| `start_line` | `Int32` | Start line number (1-based) | -| `end_line` | `Int32` | End line number (1-based) | -| `embedding` | `FixedSizeList(N)` | Model output (N = model dimension) | +| Column | Arrow Type | Description | +| ------------- | --------------------------- | ---------------------------------- | +| `chunk_id` | `Utf8` | UUID v4 | +| `file_id` | `Utf8` | FK to files.parquet | +| `chunk_index` | `Int32` | 0-based position within file | +| `start_line` | `Int32` | Start line number (1-based) | +| `end_line` | `Int32` | End line number (1-based) | +| `embedding` | `FixedSizeList(N)` | Model output (N = model dimension) | -No `plain_text` stored — model change requires re-reading files from disk. No `heading` — can be derived from file using line offsets. Line numbers are 1-based to match editor display. +No `plain_text` stored — model change requires re-reading files from disk. No +`heading` — can be derived from file using line offsets. Line numbers are +1-based to match editor display. --- @@ -209,8 +233,10 @@ crates/mfv/src/ ``` - **`cmd/`** — one file per command, each is a self-contained workflow -- **`scan/`** — reading files from disk: walking directories + extracting frontmatter -- **`report/`** — validation results: diagnostic type, validation logic, output formatting +- **`scan/`** — reading files from disk: walking directories + extracting + frontmatter +- **`report/`** — validation results: diagnostic type, validation logic, output + formatting ### `crates/mdvs/src/` (new for v0.3) diff --git a/docs/spec/commands/build.md b/docs/spec/commands/build.md index 37e4861..22d88a6 100644 --- a/docs/spec/commands/build.md +++ b/docs/spec/commands/build.md @@ -4,35 +4,76 @@ Check frontmatter, embed markdown, write the Lance index. ## Pipeline -`cmd/build/mod.rs` → `run()` → `pub async fn build_core()`. `build_core` is public so profiling examples (`crates/mdvs/examples/profile_pipeline.rs`) can drive it directly and read the per-phase `StepEntry` timings. +`cmd/build/mod.rs` → `run()` → `pub async fn build_core()`. `build_core` is +public so profiling examples (`crates/mdvs/examples/profile_pipeline.rs`) can +drive it directly and read the per-phase `StepEntry` timings. -1. **Read config** — `MdvsToml::read()` + `validate()`. Fill missing build sections (`[embedding_model]`, `[chunking]`, `[search]`) with defaults or `--set-*` values. -2. **Auto-update** — if `[build].auto_update` is true (the default), runs the inference pass and writes any newly discovered fields back to `mdvs.toml`. Disable with `--no-update` for deterministic CI builds. +1. **Read config** — `MdvsToml::read()` + `validate()`. Fill missing build + sections (`[embedding_model]`, `[chunking]`, `[search]`) with defaults or + `--set-*` values. +2. **Auto-update** — if `[build].auto_update` is true (the default), runs the + inference pass and writes any newly discovered fields back to `mdvs.toml`. + Disable with `--no-update` for deterministic CI builds. 3. **Scan** — `ScannedFiles::scan(path, &config.scan)` -4. **Validate** — same as `check`: `validate::validate()` (frontmatter errors → field values → required fields → deterministic `collect_violations`). Aborts on any violation — no dirty data in index. -5. **Classify** — compare scanned files against `FileIndexEntry` projected from the existing Lance dataset. Returns `ClassifyData` with `removed_file_ids`, `needs_embedding`, `retained_chunks`, and the file_id map: +4. **Validate** — same as `check`: `validate::validate()` (frontmatter errors → + field values → required fields → deterministic `collect_violations`). Aborts + on any violation — no dirty data in index. +5. **Classify** — compare scanned files against `FileIndexEntry` projected from + the existing Lance dataset. Returns `ClassifyData` with `removed_file_ids`, + `needs_embedding`, `retained_chunks`, and the file_id map: - New (no previous entry) → chunk + embed - Edited (hash differs) → re-chunk + re-embed (keep file_id) - Unchanged (hash matches) → retain existing chunks - Removed (in index, not in scan) → drop -6. **Config change check** — compare current `BuildMetadata` against stored. Mismatch → error unless `--force`. -7. **Load model** — `Embedder::load()` (`index/embed.rs`). Skipped if `needs_embedding == 0`. -8. **Chunk + embed** — for each new/edited file: `Chunks::new(body, max_chars)` → `embedder.embed_batch(texts)` → `Vec` +6. **Config change check** — compare current `BuildMetadata` against stored. + Mismatch → error unless `--force`. +7. **Load model** — `Embedder::load()` (`index/embed.rs`). Skipped if + `needs_embedding == 0`. +8. **Chunk + embed** — for each new/edited file: `Chunks::new(body, max_chars)` + → `embedder.embed_batch(texts)` → `Vec` 9. **Merge** — combine retained chunks (from unchanged files) with new chunks -10. **Write** — `cmd::build::write::write_index_step` dispatches across three paths: - - **Skip** when no files were removed AND no new chunks were produced AND it's not a full rebuild. Returns `WriteOutcome::Skipped`; the step appears as `Skipped` in the rendered output. The skip predicate uses `new_chunks_count` (not file count) because empty-body files like Hugo `_index.md` always classify as needing embedding but produce zero chunks. - - **Full overwrite** when `full_rebuild` is true (first build or `--force`) — `LanceBackend::write_index()` recreates the table via `CreateTableMode::Overwrite` and rebuilds the FTS + (above 10k chunks) IVF-PQ indexes. - - **Incremental** otherwise — `LanceBackend::write_index_incremental()` deletes the rows for `file_ids_to_clear` (= new + edited + removed file_ids), appends the freshly embedded chunks, refreshes the schema metadata via `NativeTable::replace_schema_metadata`, and runs `optimize(All)` so the FTS + vector indexes incorporate the delta without a full rebuild. +10. **Write** — `cmd::build::write::write_index_step` dispatches across three + paths: + - **Skip** when no files were removed AND no new chunks were produced AND + it's not a full rebuild. Returns `WriteOutcome::Skipped`; the step appears + as `Skipped` in the rendered output. The skip predicate uses + `new_chunks_count` (not file count) because empty-body files like Hugo + `_index.md` always classify as needing embedding but produce zero chunks. + - **Full overwrite** when `full_rebuild` is true (first build or `--force`) + — `LanceBackend::write_index()` recreates the table via + `CreateTableMode::Overwrite` and rebuilds the FTS + (above 10k chunks) + IVF-PQ indexes. + - **Incremental** otherwise — `LanceBackend::write_index_incremental()` + deletes the rows for `file_ids_to_clear` (= new + edited + removed + file_ids), appends the freshly embedded chunks, refreshes the schema + metadata via `NativeTable::replace_schema_metadata`, and runs + `optimize(All)` so the FTS + vector indexes incorporate the delta without + a full rebuild. Returns `BuildOutcome` with file/chunk counts, `new_fields`, `file_details`. ## Key points -- **Build includes check** — validation runs before embedding. Any violation aborts the build. -- **Incremental by default** — `content_hash` (xxh3 on body) determines what needs re-embedding. Frontmatter-only changes don't trigger re-embedding (but rewrite the `data` column on every retained chunk row). -- **Model skip** — if all files are unchanged, model loading is skipped entirely (fast no-op). -- **Write skip** — on an unchanged corpus, the index write itself is skipped too (no Lance dataset rewrite, no `optimize`, no FTS rebuild). Skipped steps render as nothing in text output but appear in `--output json` step lists as `"status": "skipped"`. -- **`--force`** — required when config changes (model, chunk_size, prefix) are detected, or when the schema content hash differs from the stored hash. The schema-hash error reads: `"schema: fields, types, constraints, path-scoping, or preprocessors have changed"`. First build (no existing Lance dataset) never needs `--force`. -- **Schema hash** — `compute_schema_hash(config)` hashes the post-translation canonical JSON of `dsl_to_canonical(config)` via xxh3-64. Stored as `mdvs.schema_hash` on the Lance table metadata. Pre-Wave-B builds without this key read as `""` → always treated as changed. +- **Build includes check** — validation runs before embedding. Any violation + aborts the build. +- **Incremental by default** — `content_hash` (xxh3 on body) determines what + needs re-embedding. Frontmatter-only changes don't trigger re-embedding (but + rewrite the `data` column on every retained chunk row). +- **Model skip** — if all files are unchanged, model loading is skipped entirely + (fast no-op). +- **Write skip** — on an unchanged corpus, the index write itself is skipped too + (no Lance dataset rewrite, no `optimize`, no FTS rebuild). Skipped steps + render as nothing in text output but appear in `--output json` step lists as + `"status": "skipped"`. +- **`--force`** — required when config changes (model, chunk_size, prefix) are + detected, or when the schema content hash differs from the stored hash. The + schema-hash error reads: + `"schema: fields, types, constraints, path-scoping, or preprocessors have changed"`. + First build (no existing Lance dataset) never needs `--force`. +- **Schema hash** — `compute_schema_hash(config)` hashes the post-translation + canonical JSON of `dsl_to_canonical(config)` via xxh3-64. Stored as + `mdvs.schema_hash` on the Lance table metadata. Pre-Wave-B builds without this + key read as `""` → always treated as changed. -See [storage.md](../storage.md) for the Lance dataset schema and incremental classification details. +See [storage.md](../storage.md) for the Lance dataset schema and incremental +classification details. diff --git a/docs/spec/commands/check.md b/docs/spec/commands/check.md index b907463..a504e9c 100644 --- a/docs/spec/commands/check.md +++ b/docs/spec/commands/check.md @@ -1,49 +1,117 @@ # `mdvs check` -Validate frontmatter against the schema. Read-only — never modifies files or config. +Validate frontmatter against the schema. Read-only — never modifies files or +config. ## Pipeline `cmd/check/mod.rs` → `run()` → `pub fn validate()` (in `validate.rs`). -1. **Resolve config** — `resolve_check_config()` reads `mdvs.toml` and, if `--jsonschema PATH` is provided, overrides the `[fields]` section. When no `mdvs.toml` exists but `--jsonschema` is given, synthesizes a default via `MdvsToml::default_with_fields(fields, ignore)` so downstream code sees a normal `MdvsToml`. -2. **Auto-update** — if `[check].auto_update` is true (default) and no `--jsonschema` override, runs inference + merges any newly-discovered fields into the config. Fields with unrepresentable shapes (`Array(Object{...})`) are partitioned out by `InferredSchema::infer` and surfaced via `emit_dropped_warnings()` to stderr; they are NOT added to the config. -3. **Scan** — `ScannedFiles::scan(path, &config.scan)` — `ScannedFile.frontmatter_error` carries any YAML→JSON representation failure. +1. **Resolve config** — `resolve_check_config()` reads `mdvs.toml` and, if + `--jsonschema PATH` is provided, overrides the `[fields]` section. When no + `mdvs.toml` exists but `--jsonschema` is given, synthesizes a default via + `MdvsToml::default_with_fields(fields, ignore)` so downstream code sees a + normal `MdvsToml`. +2. **Auto-update** — if `[check].auto_update` is true (default) and no + `--jsonschema` override, runs inference + merges any newly-discovered fields + into the config. Fields with unrepresentable shapes (`Array(Object{...})`) + are partitioned out by `InferredSchema::infer` and surfaced via + `emit_dropped_warnings()` to stderr; they are NOT added to the config. +3. **Scan** — `ScannedFiles::scan(path, &config.scan)` — + `ScannedFile.frontmatter_error` carries any YAML→JSON representation failure. 4. **Build validators + field metas + pipeline** — once per `validate()` call: - - `FieldValidators::build(config)` compiles one `jsonschema::Validator` per leaf via `dsl_to_canonical` + `extract_leaf_schemas` (keyed by dotted name). - - `build_field_metas(config)` precomputes per-field `FieldMeta` (compiled `GlobSet`s for `allowed` / `required`, plus a cached `FieldType::try_from(field.field_type)`). Without this, the inner `(field, file)` loop would call `Glob::new` and `FieldType::try_from` tens of thousands of times. - - `Pipeline::for_config(config)` builds the Stage 2 preprocessor pipeline per field. - - Per-file path strings are precomputed once so `path.display().to_string()` doesn't run inside the per-required-field inner loop. + - `FieldValidators::build(config)` compiles one `jsonschema::Validator` per + leaf via `dsl_to_canonical` + `extract_leaf_schemas` (keyed by dotted + name). + - `build_field_metas(config)` precomputes per-field `FieldMeta` (compiled + `GlobSet`s for `allowed` / `required`, plus a cached + `FieldType::try_from(field.field_type)`). Without this, the inner + `(field, file)` loop would call `Glob::new` and `FieldType::try_from` tens + of thousands of times. + - `Pipeline::for_config(config)` builds the Stage 2 preprocessor pipeline per + field. + - Per-file path strings are precomputed once so `path.display().to_string()` + doesn't run inside the per-required-field inner loop. 5. **Validate** — for each file: - - **Frontmatter errors** — if `frontmatter_error` is set, emit `ViolationKind::FrontmatterUnrepresentable` with sentinel field ``. - - **Per-field values** — `check_field_values` runs the Stage 2 pipeline to normalize, then takes a `validator.is_valid()` fast path: when no errors exist (the common case), only the path-scoping `Disallowed` check fires. Only fields whose validator returned errors go through the full `validator.iter_errors()` + `map_validation_error` path. The strict-Float precheck (`preprocess::strict_subtype_check`) runs before either of the above. Path-scoping uses the precomputed `FieldMeta::allowed` `GlobSet`. - - **Required fields** — `check_required_fields` per-field iteration over the precomputed `FieldMeta::required` `GlobSet`. -6. **Collect** — `collect_violations()` groups by `ViolationKey { field, kind, rule }`. Output sort is byte-stable: outer sort `(field, kind, rule)` where `kind` uses `ViolationKind`'s `Ord` derive (declaration order: `MissingRequired` < `WrongType` < `Disallowed` < `NullNotAllowed` < `InvalidCategory` < `OutOfRange` < `FrontmatterUnrepresentable`); inner sort files within each violation by path. The byte-stability contract matters for CI consumers that diff `mdvs check` output across runs. + - **Frontmatter errors** — if `frontmatter_error` is set, emit + `ViolationKind::FrontmatterUnrepresentable` with sentinel field + ``. + - **Per-field values** — `check_field_values` runs the Stage 2 pipeline to + normalize, then takes a `validator.is_valid()` fast path: when no errors + exist (the common case), only the path-scoping `Disallowed` check fires. + Only fields whose validator returned errors go through the full + `validator.iter_errors()` + `map_validation_error` path. The strict-Float + precheck (`preprocess::strict_subtype_check`) runs before either of the + above. Path-scoping uses the precomputed `FieldMeta::allowed` `GlobSet`. + - **Required fields** — `check_required_fields` per-field iteration over the + precomputed `FieldMeta::required` `GlobSet`. +6. **Collect** — `collect_violations()` groups by + `ViolationKey { field, kind, rule }`. Output sort is byte-stable: outer sort + `(field, kind, rule)` where `kind` uses `ViolationKind`'s `Ord` derive + (declaration order: `MissingRequired` < `WrongType` < `Disallowed` < + `NullNotAllowed` < `InvalidCategory` < `OutOfRange` < + `FrontmatterUnrepresentable`); inner sort files within each violation by + path. The byte-stability contract matters for CI consumers that diff + `mdvs check` output across runs. -Returns `CheckOutcome` with `files_checked`, `violations: Vec`, `new_fields: Vec`. +Returns `CheckOutcome` with `files_checked`, `violations: Vec`, +`new_fields: Vec`. ## Validation engine (post-Wave-B) -Validation runs through the `jsonschema` crate (v0.46). Hand-rolled per-value validators have been removed. +Validation runs through the `jsonschema` crate (v0.46). Hand-rolled per-value +validators have been removed. -- **Translation** — `dsl_to_canonical(config)` translates `[fields]` into a JSON Schema 2020-12 document. Per-field validators are compiled once per `validate()` call, keyed by the field's full dotted name (e.g. `calibration.baseline.wavelength`). Extracted from the canonical schema's nested `properties` tree via `extract_leaf_schemas` (TODO-0097 step 4). -- **Dotted-path navigation** — `navigate_dotted(frontmatter, "cal.baseline.wave")` walks the YAML's nested Object structure to retrieve the leaf value. An absent intermediate counts as the leaf being absent (handled by `check_required_fields`). -- **Strict subtype precheck** — `preprocess::strict_subtype_check` runs in Rust before the preprocessor pipeline. Currently enforces strict-Float (rejects integer-backed values on Float / Array(Float) fields unless `widen-int-to-float` is in `preprocess`). See [architecture.md](../architecture.md#strict-subtype-prechecks) for the rationale. -- **Preprocessing** — each field's `preprocess` array (e.g. `["coerce-to-string"]`) runs before jsonschema, transforming the value via `Pipeline::apply_to_value`. -- **Format validation** — Validators are built with `jsonschema::options().should_validate_formats(true)`. Two formats are validated at runtime: `date` (RFC 3339 full-date) and `date-time` (RFC 3339 datetime). Other format values are rejected by the schema gate, so they can't reach the validator. Format failures map to `WrongType` with rule `format ` (TODO-0007). -- **Error mapping** — `map_validation_error` is an exhaustive match over `ValidationErrorKind`; new variants in future jsonschema versions cause a compile error rather than a silent fallback. -- **Array-of-mappings against a scalar `Array` field** — fires the existing `WrongType` violation (the element is a JSON Object, not the expected scalar type). No special `Array(Object)` handling is needed in validation because the on-disk type vocabulary doesn't include it (TODO-0155). +- **Translation** — `dsl_to_canonical(config)` translates `[fields]` into a JSON + Schema 2020-12 document. Per-field validators are compiled once per + `validate()` call, keyed by the field's full dotted name (e.g. + `calibration.baseline.wavelength`). Extracted from the canonical schema's + nested `properties` tree via `extract_leaf_schemas` (TODO-0097 step 4). +- **Dotted-path navigation** — + `navigate_dotted(frontmatter, "cal.baseline.wave")` walks the YAML's nested + Object structure to retrieve the leaf value. An absent intermediate counts as + the leaf being absent (handled by `check_required_fields`). +- **Strict subtype precheck** — `preprocess::strict_subtype_check` runs in Rust + before the preprocessor pipeline. Currently enforces strict-Float (rejects + integer-backed values on Float / Array(Float) fields unless + `widen-int-to-float` is in `preprocess`). See + [architecture.md](../architecture.md#strict-subtype-prechecks) for the + rationale. +- **Preprocessing** — each field's `preprocess` array (e.g. + `["coerce-to-string"]`) runs before jsonschema, transforming the value via + `Pipeline::apply_to_value`. +- **Format validation** — Validators are built with + `jsonschema::options().should_validate_formats(true)`. Two formats are + validated at runtime: `date` (RFC 3339 full-date) and `date-time` (RFC 3339 + datetime). Other format values are rejected by the schema gate, so they can't + reach the validator. Format failures map to `WrongType` with rule + `format ` (TODO-0007). +- **Error mapping** — `map_validation_error` is an exhaustive match over + `ValidationErrorKind`; new variants in future jsonschema versions cause a + compile error rather than a silent fallback. +- **Array-of-mappings against a scalar `Array` field** — fires the existing + `WrongType` violation (the element is a JSON Object, not the expected scalar + type). No special `Array(Object)` handling is needed in validation because the + on-disk type vocabulary doesn't include it (TODO-0155). -See [architecture.md](../architecture.md#validation-pipeline) for the full pipeline and error mapping table. +See [architecture.md](../architecture.md#validation-pipeline) for the full +pipeline and error mapping table. ## `--jsonschema` override -`mdvs check --jsonschema PATH` replaces the `[fields]` block for this run. Useful for one-off validation against a contract without editing `mdvs.toml`. The file is loaded via `schema/load.rs` (extension-dispatched: `.json` / `.toml`) and gated via `validate_mdvs_schema`. +`mdvs check --jsonschema PATH` replaces the `[fields]` block for this run. +Useful for one-off validation against a contract without editing `mdvs.toml`. +The file is loaded via `schema/load.rs` (extension-dispatched: `.json` / +`.toml`) and gated via `validate_mdvs_schema`. ## Violation grouping -`ViolationKey { field, kind, rule }` groups files violating the same rule. Multiple files with the same violation → one `FieldViolation` entry with `files: Vec`. Detail (e.g., `got String`) lives on `ViolatingFile`, not the key. +`ViolationKey { field, kind, rule }` groups files violating the same rule. +Multiple files with the same violation → one `FieldViolation` entry with +`files: Vec`. Detail (e.g., `got String`) lives on +`ViolatingFile`, not the key. ## ViolationKind values -`MissingRequired`, `WrongType`, `Disallowed`, `NullNotAllowed`, `InvalidCategory`, `OutOfRange`, `FrontmatterUnrepresentable`. +`MissingRequired`, `WrongType`, `Disallowed`, `NullNotAllowed`, +`InvalidCategory`, `OutOfRange`, `FrontmatterUnrepresentable`. diff --git a/docs/spec/commands/clean.md b/docs/spec/commands/clean.md index d63356d..2c26726 100644 --- a/docs/spec/commands/clean.md +++ b/docs/spec/commands/clean.md @@ -10,10 +10,12 @@ Delete the `.mdvs/` index directory. 2. **Stats** — `walk_dir_stats()` counts files and sums sizes for the outcome 3. **Delete** — `fs::remove_dir_all()` -Returns `CleanOutcome` with `removed: bool`, `path`, `files_removed`, `size_bytes`. +Returns `CleanOutcome` with `removed: bool`, `path`, `files_removed`, +`size_bytes`. ## Key points -- **Destructive** — removes the Lance dataset (`index.lance/`), the cached embedding model, and build metadata. Requires a `build` to recreate. +- **Destructive** — removes the Lance dataset (`index.lance/`), the cached + embedding model, and build metadata. Requires a `build` to recreate. - **Config preserved** — `mdvs.toml` is not touched. Only `.mdvs/` is deleted. - **No confirmation** — deletes immediately. No `--force` required. diff --git a/docs/spec/commands/export-jsonschema.md b/docs/spec/commands/export-jsonschema.md index bc0ed45..6b25c7b 100644 --- a/docs/spec/commands/export-jsonschema.md +++ b/docs/spec/commands/export-jsonschema.md @@ -1,50 +1,66 @@ # `mdvs export-jsonschema` -Translate the `[fields]` block of `mdvs.toml` into a JSON Schema 2020-12 document. Round-trips losslessly with `mdvs init --from-jsonschema`. +Translate the `[fields]` block of `mdvs.toml` into a JSON Schema 2020-12 +document. Round-trips losslessly with `mdvs init --from-jsonschema`. ## Pipeline `cmd/export_jsonschema.rs` → `run(path, format, output_file)` 1. **Read config** — `MdvsToml::read(path)` + `validate()` -2. **Translate** — `dsl_to_canonical(&config)` (`schema/json_schema.rs`) emits a JSON Schema 2020-12 document -3. **Serialize** — `--format json` (default) uses `serde_json::to_string_pretty`; `--format toml` uses `tomljson::to_string` +2. **Translate** — `dsl_to_canonical(&config)` (`schema/json_schema.rs`) emits a + JSON Schema 2020-12 document +3. **Serialize** — `--format json` (default) uses + `serde_json::to_string_pretty`; `--format toml` uses `tomljson::to_string` 4. **Write** — to `--output-file FILE` if provided, else stdout -Returns `ExportJsonschemaOutcome` with destination + format. When writing to stdout, the summary block is suppressed so the output is directly pipeable. +Returns `ExportJsonschemaOutcome` with destination + format. When writing to +stdout, the summary block is suppressed so the output is directly pipeable. ## Flags -| Flag | Default | Behavior | -|------|---------|----------| -| `[PATH]` | `.` | Project directory containing `mdvs.toml` | -| `--format json\|toml` | `json` | Output format. `toml` uses the workspace `tomljson` crate | -| `--output-file FILE` | (stdout) | Write to a file instead of stdout | +| Flag | Default | Behavior | +| --------------------- | -------- | --------------------------------------------------------- | +| `[PATH]` | `.` | Project directory containing `mdvs.toml` | +| `--format json\|toml` | `json` | Output format. `toml` uses the workspace `tomljson` crate | +| `--output-file FILE` | (stdout) | Write to a file instead of stdout | ## Extension keys -mdvs-specific metadata that JSON Schema 2020-12 doesn't model is carried in `x-mdvs` extension objects: +mdvs-specific metadata that JSON Schema 2020-12 doesn't model is carried in +`x-mdvs` extension objects: -- **Schema level** — `x-mdvs.preprocess` (top-level preprocessor stages, reserved), `x-mdvs.definitions` -- **Property level** — `x-mdvs.allowed` (path-scoping globs), `x-mdvs.required` (path-scoping globs), `x-mdvs.preprocess` (Stage 2 preprocessor list for this field) +- **Schema level** — `x-mdvs.preprocess` (top-level preprocessor stages, + reserved), `x-mdvs.definitions` +- **Property level** — `x-mdvs.allowed` (path-scoping globs), `x-mdvs.required` + (path-scoping globs), `x-mdvs.preprocess` (Stage 2 preprocessor list for this + field) -These keys are ignored by generic JSON Schema validators and round-tripped by `canonical_to_dsl`. +These keys are ignored by generic JSON Schema validators and round-tripped by +`canonical_to_dsl`. ## Round-trip guarantee -`mdvs export-jsonschema ./project --output-file out.json` followed by `mdvs init --from-jsonschema out.json ./reborn` reproduces the original `[[fields.field]]` definitions including: +`mdvs export-jsonschema ./project --output-file out.json` followed by +`mdvs init --from-jsonschema out.json ./reborn` reproduces the original +`[[fields.field]]` definitions including: - Field types (strict — `String` ≠ permissive set) -- Constraints (`categories`, `min`, `max`, `min_length`, `max_length`, `pattern`) +- Constraints (`categories`, `min`, `max`, `min_length`, `max_length`, + `pattern`) - Path-scoping (`allowed`, `required`) - Preprocessor arrays (`preprocess`) - The `[fields].ignore` list (carried in `x-mdvs.definitions`) -Build sections (`[embedding_model]`, `[chunking]`, `[search]`) and scan config are **not** exported — JSON Schema is only the fields contract. +Build sections (`[embedding_model]`, `[chunking]`, `[search]`) and scan config +are **not** exported — JSON Schema is only the fields contract. ## Not exported -- `[scan]`, `[embedding_model]`, `[chunking]`, `[search]`, `[update]`, `[check]`, `[build]` — these live in `mdvs.toml` only. -- Inference thresholds in `[fields]` (`max_distinct_for_categorical`, etc.) — these are inference hyperparameters, not part of the schema. +- `[scan]`, `[embedding_model]`, `[chunking]`, `[search]`, `[update]`, + `[check]`, `[build]` — these live in `mdvs.toml` only. +- Inference thresholds in `[fields]` (`max_distinct_for_categorical`, etc.) — + these are inference hyperparameters, not part of the schema. -See [architecture.md](../architecture.md#validation-pipeline) for the translator and gate. +See [architecture.md](../architecture.md#validation-pipeline) for the translator +and gate. diff --git a/docs/spec/commands/info.md b/docs/spec/commands/info.md index b6f8083..1d94a5c 100644 --- a/docs/spec/commands/info.md +++ b/docs/spec/commands/info.md @@ -7,15 +7,22 @@ Display project configuration and index status. `cmd/info.rs` → `run()` 1. **Read config** — `MdvsToml::read()` + `validate()` -2. **Auto-update** — if `[check].auto_update` is true, runs `update::run()` first +2. **Auto-update** — if `[check].auto_update` is true, runs `update::run()` + first 3. **Scan** — `ScannedFiles::scan()` for field prevalence counts -4. **Read index** — if `.mdvs/` exists, read `BuildMetadata` and `IndexStats` (file count, chunk count) -5. **Build field list** — for each `TomlField`: name, type, allowed, required, nullable, file count, hints +4. **Read index** — if `.mdvs/` exists, read `BuildMetadata` and `IndexStats` + (file count, chunk count) +5. **Build field list** — for each `TomlField`: name, type, allowed, required, + nullable, file count, hints -Returns `InfoOutcome` with `scan_glob`, `files_on_disk`, `fields: Vec`, `ignored_fields`, `index: Option`. +Returns `InfoOutcome` with `scan_glob`, `files_on_disk`, +`fields: Vec`, `ignored_fields`, `index: Option`. ## Key points - **Read-only** — never modifies config or index. -- **Field hints** — `FieldHint` enum detects special characters in field names (single quotes, double quotes, spaces) and suggests escaping for `--where` queries. -- **Index section** — shows model name, revision, chunk size, file/chunk counts, and build timestamp. Absent if `.mdvs/` doesn't exist. +- **Field hints** — `FieldHint` enum detects special characters in field names + (single quotes, double quotes, spaces) and suggests escaping for `--where` + queries. +- **Index section** — shows model name, revision, chunk size, file/chunk counts, + and build timestamp. Absent if `.mdvs/` doesn't exist. diff --git a/docs/spec/commands/init.md b/docs/spec/commands/init.md index 79821bf..1a956ac 100644 --- a/docs/spec/commands/init.md +++ b/docs/spec/commands/init.md @@ -1,37 +1,74 @@ # `mdvs init` -Scan a directory, infer a typed schema, and write `mdvs.toml`. Optionally import the schema from an external JSON Schema file via `--from-jsonschema`. +Scan a directory, infer a typed schema, and write `mdvs.toml`. Optionally import +the schema from an external JSON Schema file via `--from-jsonschema`. ## Pipeline (default: infer from scan) `cmd/init.rs` → `run()` -1. **Pre-checks** — directory exists, config path resolved, `--force` deletes existing config + `.mdvs/` +1. **Pre-checks** — directory exists, config path resolved, `--force` deletes + existing config + `.mdvs/` 2. **Scan** — `ScannedFiles::scan(path, &scan_config)` (`discover/scan.rs`) -3. **Infer** — `InferredSchema::infer(&scanned)` (`discover/infer/mod.rs`) — type widening, path inference, distinct value collection, observed-types tracking. Fields with unrepresentable shapes (`Array(Object{...})`) are partitioned into `schema.dropped`. -4. **Warn** — `schema.emit_dropped_warnings()` prints one stderr line per dropped field with the field name, reason, and first-observed file path. -5. **Build config** — `MdvsToml::from_inferred(&schema, scan_config)` (`schema/config.rs`) — converts representable `InferredField`s to `TomlField`s, runs `infer_constraints()` for categorical fields, runs `infer_value_stages()` to populate `preprocess` arrays from observed widening events -6. **Write** — `config.write(&path)` — serializes TOML; `type` is written as a function-style string (`"Array(String)"`). +3. **Infer** — `InferredSchema::infer(&scanned)` (`discover/infer/mod.rs`) — + type widening, path inference, distinct value collection, observed-types + tracking. Fields with unrepresentable shapes (`Array(Object{...})`) are + partitioned into `schema.dropped`. +4. **Warn** — `schema.emit_dropped_warnings()` prints one stderr line per + dropped field with the field name, reason, and first-observed file path. +5. **Build config** — `MdvsToml::from_inferred(&schema, scan_config)` + (`schema/config.rs`) — converts representable `InferredField`s to + `TomlField`s, runs `infer_constraints()` for categorical fields, runs + `infer_value_stages()` to populate `preprocess` arrays from observed widening + events +6. **Write** — `config.write(&path)` — serializes TOML; `type` is written as a + function-style string (`"Array(String)"`). -Returns `InitOutcome` with `files_scanned`, `fields: Vec`, `dry_run`. +Returns `InitOutcome` with `files_scanned`, `fields: Vec`, +`dry_run`. ## Pipeline (--from-jsonschema PATH) -Skips scan + infer; the external file is the source of truth for fields. `init_from_schema()` in `cmd/init.rs`: +Skips scan + infer; the external file is the source of truth for fields. +`init_from_schema()` in `cmd/init.rs`: -1. **Load** — `load_schema(path)` from `schema/load.rs` parses by extension (`.json` via `serde_json`, `.toml` via `tomljson`) -2. **Gate** — `validate_mdvs_schema(&schema)` (`schema/json_schema.rs`) checks the allow-list of supported keywords and rejects unsupported features (`oneOf`, `$ref`, `format`, etc.) with explanatory messages -3. **Translate** — `canonical_to_dsl(&schema)` translates JSON Schema 2020-12 back into `Vec` + ignore list, reading `x-mdvs.allowed`, `x-mdvs.required`, `x-mdvs.preprocess` -4. **Assemble** — `MdvsToml::default_with_fields(fields, ignore)` synthesizes a minimal config (no `[embedding_model]`/`[chunking]`/`[search]` build sections unless `--auto-build`) +1. **Load** — `load_schema(path)` from `schema/load.rs` parses by extension + (`.json` via `serde_json`, `.toml` via `tomljson`) +2. **Gate** — `validate_mdvs_schema(&schema)` (`schema/json_schema.rs`) checks + the allow-list of supported keywords and rejects unsupported features + (`oneOf`, `$ref`, `format`, etc.) with explanatory messages +3. **Translate** — `canonical_to_dsl(&schema)` translates JSON Schema 2020-12 + back into `Vec` + ignore list, reading `x-mdvs.allowed`, + `x-mdvs.required`, `x-mdvs.preprocess` +4. **Assemble** — `MdvsToml::default_with_fields(fields, ignore)` synthesizes a + minimal config (no `[embedding_model]`/`[chunking]`/`[search]` build sections + unless `--auto-build`) 5. **Write** — same as default flow ## Key points -- **Schema-only** — init never downloads a model, never creates `.mdvs/`, never embeds. -- **Categorical inference** — `infer_constraints()` runs with defaults (max_categories=10, min_repetition=3). Qualifying fields get `[fields.field.constraints].categories`. -- **Date / DateTime inference** — string values matching RFC 3339 full-date (`YYYY-MM-DD`) auto-infer as `Date`; values matching RFC 3339 datetime (`YYYY-MM-DDTHH:MM:SS[.frac]`) auto-infer as `DateTime`. A single non-matching observation downgrades the field to `String`. See [TODO-0007](../todos/TODO-0007.md) and `book/src/concepts/types.md#date-and-datetime`. -- **Preprocessor inference** — observed type-widening events drive `[fields.field].preprocess`. No implicit defaults: `preprocess = []` means strict. -- **`init --force` vs `update reinfer`** — init rewrites the entire config (all sections). `update reinfer` re-infers only `[fields]`, preserving all other config. -- **Round-trip with `mdvs export-jsonschema`** — exporting then re-importing reproduces the original `[[fields.field]]` definitions including constraints, path-scoping, and `preprocess` arrays (preserved via `x-mdvs.*` extension keys). +- **Schema-only** — init never downloads a model, never creates `.mdvs/`, never + embeds. +- **Categorical inference** — `infer_constraints()` runs with defaults + (max_categories=10, min_repetition=3). Qualifying fields get + `[fields.field.constraints].categories`. +- **Date / DateTime inference** — string values matching RFC 3339 full-date + (`YYYY-MM-DD`) auto-infer as `Date`; values matching RFC 3339 datetime + (`YYYY-MM-DDTHH:MM:SS[.frac]`) auto-infer as `DateTime`. A single + non-matching observation downgrades the field to `String`. See + [TODO-0007](../todos/TODO-0007.md) and + `book/src/concepts/types.md#date-and-datetime`. +- **Preprocessor inference** — observed type-widening events drive + `[fields.field].preprocess`. No implicit defaults: `preprocess = []` means + strict. +- **`init --force` vs `update reinfer`** — init rewrites the entire config (all + sections). `update reinfer` re-infers only `[fields]`, preserving all other + config. +- **Round-trip with `mdvs export-jsonschema`** — exporting then re-importing + reproduces the original `[[fields.field]]` definitions including constraints, + path-scoping, and `preprocess` arrays (preserved via `x-mdvs.*` extension + keys). -See [inference.md](../inference.md) for the inference algorithm and [architecture.md](../architecture.md#validation-pipeline) for the translation gates. +See [inference.md](../inference.md) for the inference algorithm and +[architecture.md](../architecture.md#validation-pipeline) for the translation +gates. diff --git a/docs/spec/commands/search.md b/docs/spec/commands/search.md index ad38679..d9aa489 100644 --- a/docs/spec/commands/search.md +++ b/docs/spec/commands/search.md @@ -1,34 +1,57 @@ # `mdvs search` -Query the Lance index via LanceDB — semantic (cosine), full-text (BM25), or hybrid (RRF reranker over both). +Query the Lance index via LanceDB — semantic (cosine), full-text (BM25), or +hybrid (RRF reranker over both). ## Pipeline `cmd/search.rs` → `run()` 1. **Read config** — `MdvsToml::read()` + `validate()` -2. **Auto-update + auto-build** — chains update → build if configured (respects `[search].auto_update` and `[search].auto_build`) -3. **Read index metadata** — `LanceBackend::read_metadata()`. Hard error if the model in config differs from the model stored on the Lance table metadata. -4. **Load model** — `Embedder::load()` from config. Skipped for `--mode fulltext`. -5. **Embed query** — `embedder.embed(&query)` → `Vec`. Skipped for `--mode fulltext`. -6. **Execute search** — `LanceBackend::search(mode, query, query_embedding, where_clause, limit)`: - - Translates `--where` via `translate_where_to_struct` (bare frontmatter names → `data.*`; scalar function calls left as-is; references to `Array(Float)` fields rejected — see TODO-0159). +2. **Auto-update + auto-build** — chains update → build if configured (respects + `[search].auto_update` and `[search].auto_build`) +3. **Read index metadata** — `LanceBackend::read_metadata()`. Hard error if the + model in config differs from the model stored on the Lance table metadata. +4. **Load model** — `Embedder::load()` from config. Skipped for + `--mode fulltext`. +5. **Embed query** — `embedder.embed(&query)` → `Vec`. Skipped for + `--mode fulltext`. +6. **Execute search** — + `LanceBackend::search(mode, query, query_embedding, where_clause, limit)`: + - Translates `--where` via `translate_where_to_struct` (bare frontmatter + names → `data.*`; scalar function calls left as-is; references to + `Array(Float)` fields rejected — see TODO-0159). - Builds the LanceDB query for the selected `SearchMode`: - `Semantic` → `.nearest_to(query_embedding).distance_type(Cosine)` - `Fulltext` → `.full_text_search(FullTextSearchQuery::new(query))` - - `Hybrid` (default) → both of the above + `.rerank(RrfReranker::default())` - - Applies `.only_if()` if a filter was given and `.limit(limit × OVER_FETCH_FACTOR)` (`OVER_FETCH_FACTOR = 3`). - - Streams chunk rows back; reads the per-mode score column (`_distance` mapped to `1 - d`, `_score`, or `_relevance_score`). -7. **Best-chunk-per-file dedupe** — group by `file_id`, keep the highest-scored chunk per file, trim to `--limit`. -8. **Verbose snippet** — read directly from the persisted `chunk_text` column on the winning row (no second file read). - -Returns `SearchOutcome` with `query`, `mode`, `hits: Vec`, `model_name`, `limit`. + - `Hybrid` (default) → both of the above + + `.rerank(RrfReranker::default())` + - Applies `.only_if()` if a filter was given and + `.limit(limit × OVER_FETCH_FACTOR)` (`OVER_FETCH_FACTOR = 3`). + - Streams chunk rows back; reads the per-mode score column (`_distance` + mapped to `1 - d`, `_score`, or `_relevance_score`). +7. **Best-chunk-per-file dedupe** — group by `file_id`, keep the highest-scored + chunk per file, trim to `--limit`. +8. **Verbose snippet** — read directly from the persisted `chunk_text` column on + the winning row (no second file read). + +Returns `SearchOutcome` with `query`, `mode`, `hits: Vec`, +`model_name`, `limit`. ## Key points -- **Model mismatch → hard error** — applies to semantic and hybrid modes; fulltext doesn't load the model and is unaffected. -- **Note-level ranking** — results are per-file, scored by the best chunk (max, not average). -- **`--where` translation** — bare frontmatter names get a `data.` prefix, scalar function calls (`lower(...)`, `length(...)`, `abs(...)`, …) are left untouched, and `Array(Float)` field references produce an early error before LanceDB sees them. Dot-notation works for nested-leaf fields (`WHERE data.calibration.baseline.wavelength > 800`). -- **No vector index below 10k chunks** — `VECTOR_INDEX_MIN_ROWS = 10_000` gates IVF-PQ. Smaller vaults run an exact flat scan inside LanceDB, which is plenty fast at that scale. - -See [search.md](../search.md) for the `SearchMode` dispatch, score-column resolution, and dedupe details. +- **Model mismatch → hard error** — applies to semantic and hybrid modes; + fulltext doesn't load the model and is unaffected. +- **Note-level ranking** — results are per-file, scored by the best chunk (max, + not average). +- **`--where` translation** — bare frontmatter names get a `data.` prefix, + scalar function calls (`lower(...)`, `length(...)`, `abs(...)`, …) are left + untouched, and `Array(Float)` field references produce an early error before + LanceDB sees them. Dot-notation works for nested-leaf fields + (`WHERE data.calibration.baseline.wavelength > 800`). +- **No vector index below 10k chunks** — `VECTOR_INDEX_MIN_ROWS = 10_000` gates + IVF-PQ. Smaller vaults run an exact flat scan inside LanceDB, which is plenty + fast at that scale. + +See [search.md](../search.md) for the `SearchMode` dispatch, score-column +resolution, and dedupe details. diff --git a/docs/spec/commands/update.md b/docs/spec/commands/update.md index 782419e..bbbd8b7 100644 --- a/docs/spec/commands/update.md +++ b/docs/spec/commands/update.md @@ -1,37 +1,58 @@ # `mdvs update` -Re-scan files, infer field changes, and update `mdvs.toml`. Pure inference — no build step. +Re-scan files, infer field changes, and update `mdvs.toml`. Pure inference — no +build step. ## Pipeline `cmd/update.rs` → `run()` 1. **Read config** — `MdvsToml::read()` + `validate()` -2. **Pre-check** — validate `--with` requires named fields; validate `--with` value list (no `none` mixed with other kinds; pairwise compatibility via `with_kinds_conflict()`) +2. **Pre-check** — validate `--with` requires named fields; validate `--with` + value list (no `none` mixed with other kinds; pairwise compatibility via + `with_kinds_conflict()`) 3. **Scan** — `ScannedFiles::scan(path, &config.scan)` -4. **Infer** — `InferredSchema::infer(&scanned)` — full inference (types, paths, distinct values). Fields with unrepresentable shapes (`Array(Object{...})`) are partitioned into `schema.dropped`; `emit_dropped_warnings()` prints one stderr line per dropped field. Date and DateTime are auto-detected from RFC 3339 strings — see [TODO-0007](../todos/TODO-0007.md). -5. **Partition** — split config fields into `protected` (keep) and `targets` (reinfer): +4. **Infer** — `InferredSchema::infer(&scanned)` — full inference (types, paths, + distinct values). Fields with unrepresentable shapes (`Array(Object{...})`) + are partitioned into `schema.dropped`; `emit_dropped_warnings()` prints one + stderr line per dropped field. Date and DateTime are auto-detected from RFC + 3339 strings — see [TODO-0007](../todos/TODO-0007.md). +5. **Partition** — split config fields into `protected` (keep) and `targets` + (reinfer): - No reinfer → all protected, empty targets (only new fields discovered) - `reinfer field1 field2` → named fields are targets, rest protected - `reinfer` (no fields) → all are targets -6. **Compare** — for each inferred field: if protected → skip; if in ignore → skip; else construct `TomlField` with constraints, compare against old definition → added/changed/unchanged/removed -7. **Write** — update `config.fields.field` with new list, write TOML (unless dry_run or no changes) +6. **Compare** — for each inferred field: if protected → skip; if in ignore → + skip; else construct `TomlField` with constraints, compare against old + definition → added/changed/unchanged/removed +7. **Write** — update `config.fields.field` with new list, write TOML (unless + dry_run or no changes) -Returns `UpdateOutcome` with `files_scanned`, `added`, `changed`, `removed`, `unchanged`, `dry_run`. +Returns `UpdateOutcome` with `files_scanned`, `added`, `changed`, `removed`, +`unchanged`, `dry_run`. ## Constraint inference in reinfer -The `ReinferArgs.with: Vec` field drives constraint construction. `WithKind` is a CLI-local enum with variants `Categorical`, `Range`, `None`. +The `ReinferArgs.with: Vec` field drives constraint construction. +`WithKind` is a CLI-local enum with variants `Categorical`, `Range`, `None`. When constructing `TomlField` for reinferred fields: -- **No reinfer** (default mode) → `constraints: None` (no constraint changes on new fields) +- **No reinfer** (default mode) → `constraints: None` (no constraint changes on + new fields) - **`with` contains `None`** → `constraints: None` (strip all) -- **`with` is empty** → `infer_constraints(&inf, max_cat, min_rep)` (heuristic default — currently categorical only) +- **`with` is empty** → `infer_constraints(&inf, max_cat, min_rep)` (heuristic + default — currently categorical only) - **`with` is non-empty (no `None`)** → for each kind, force-infer: - - `Categorical` → `force_categorical(&inf)` (all distinct values as categories, skip heuristic threshold) + - `Categorical` → `force_categorical(&inf)` (all distinct values as + categories, skip heuristic threshold) - `Range` → `infer_range(&inf)` (min/max from observed numeric values) -`force_categorical()` and `infer_range()` are in `cmd/update.rs` and `discover/infer/constraints/range.rs` respectively. Both check type applicability before producing values. +`force_categorical()` and `infer_range()` are in `cmd/update.rs` and +`discover/infer/constraints/range.rs` respectively. Both check type +applicability before producing values. -`with_kinds_conflict()` in `cmd/update.rs` defines pairwise CLI-level incompatibility (currently only `Categorical` ↔ `Range`). This is independent of the deeper `ConstraintKind::conflicts_with()` validation that runs at config load time, but the rules align. +`with_kinds_conflict()` in `cmd/update.rs` defines pairwise CLI-level +incompatibility (currently only `Categorical` ↔ `Range`). This is independent +of the deeper `ConstraintKind::conflicts_with()` validation that runs at config +load time, but the rules align. diff --git a/docs/spec/inference.md b/docs/spec/inference.md index 2370f44..028ffef 100644 --- a/docs/spec/inference.md +++ b/docs/spec/inference.md @@ -1,28 +1,37 @@ # Inference -Deep-dive into the schema inference subsystem. For the module map see [architecture.md](./architecture.md). +Deep-dive into the schema inference subsystem. For the module map see +[architecture.md](./architecture.md). -Inference spans four file groups: type inference (`discover/infer/types.rs` + `discover/field_type.rs`), path inference (`discover/infer/paths.rs`), constraint inference (`discover/infer/constraints/`), and the orchestrator (`discover/infer/mod.rs`). +Inference spans four file groups: type inference (`discover/infer/types.rs` + +`discover/field_type.rs`), path inference (`discover/infer/paths.rs`), +constraint inference (`discover/infer/constraints/`), and the orchestrator +(`discover/infer/mod.rs`). ## Type Widening -`FieldType::from_widen(a, b)` at `discover/field_type.rs:29` computes the least upper bound of two types. The operation is symmetric: `widen(A, B) == widen(B, A)`. +`FieldType::from_widen(a, b)` at `discover/field_type.rs:29` computes the least +upper bound of two types. The operation is symmetric: +`widen(A, B) == widen(B, A)`. -| Type A | Type B | Result | Rule | -|--------|--------|--------|------| -| same | same | same | identity | -| Integer | Float | Float | numeric promotion | -| Array(T1) | Array(T2) | Array(widen(T1, T2)) | recursive | -| Object(K1) | Object(K2) | Object(merged) | union keys, widen shared | -| *anything else* | *anything else* | String | fallback (top type) | +| Type A | Type B | Result | Rule | +| --------------- | --------------- | -------------------- | ------------------------ | +| same | same | same | identity | +| Integer | Float | Float | numeric promotion | +| Array(T1) | Array(T2) | Array(widen(T1, T2)) | recursive | +| Object(K1) | Object(K2) | Object(merged) | union keys, widen shared | +| _anything else_ | _anything else_ | String | fallback (top type) | -Object merging: all keys from both sides are kept. Shared keys are widened recursively. Unique keys pass through unchanged. +Object merging: all keys from both sides are kept. Shared keys are widened +recursively. Unique keys pass through unchanged. -The "anything else" fallback covers: Boolean+Integer, Boolean+Float, Boolean+String, scalar+Array, scalar+Object, Array+Object. All widen to String. +The "anything else" fallback covers: Boolean+Integer, Boolean+Float, +Boolean+String, scalar+Array, scalar+Object, Array+Object. All widen to String. ## Type Detection -`From<&Value> for FieldType` at `discover/field_type.rs:61` maps JSON values to types: +`From<&Value> for FieldType` at `discover/field_type.rs:61` maps JSON values to +types: - `Number` with `is_i64() || is_u64()` → Integer, otherwise Float - Empty array `[]` → Array(String) (placeholder) @@ -32,21 +41,30 @@ The "anything else" fallback covers: Boolean+Integer, Boolean+Float, Boolean+Str ## Null Transparency -In `infer_field_types()` at `discover/infer/types.rs:28`, null values are **transparent** in type widening: +In `infer_field_types()` at `discover/infer/types.rs:28`, null values are +**transparent** in type widening: -1. **Skip in widening** — null values hit `continue` at line 45, never entering the `from_widen()` call. A field with `null` in file A and `42` in file B infers as Integer, not String. -2. **Track separately** — `nulls: HashSet` records which fields had any null value. This becomes `nullable: true` on the `FieldTypeInfo`. -3. **Default for null-only** — fields appearing only as null (never a real value) default to String at line 66: `types.entry(key).or_insert(FieldType::String)`. +1. **Skip in widening** — null values hit `continue` at line 45, never entering + the `from_widen()` call. A field with `null` in file A and `42` in file B + infers as Integer, not String. +2. **Track separately** — `nulls: HashSet` records which fields had any + null value. This becomes `nullable: true` on the `FieldTypeInfo`. +3. **Default for null-only** — fields appearing only as null (never a real + value) default to String at line 66: + `types.entry(key).or_insert(FieldType::String)`. -File presence is always tracked regardless of null — a null-valued field counts as "present" for allowed/required glob computation. +File presence is always tracked regardless of null — a null-valued field counts +as "present" for allowed/required glob computation. ## Path Inference -The hardest algorithm in the codebase. Converts per-file field presence into glob patterns. +The hardest algorithm in the codebase. Converts per-file field presence into +glob patterns. ### Tree Construction -`DirectoryTree::from(scanned)` at `discover/infer/paths.rs:37` builds an arena-based tree: +`DirectoryTree::from(scanned)` at `discover/infer/paths.rs:37` builds an +arena-based tree: 1. Create root node (empty path) 2. For each file, extract the set of frontmatter field names @@ -57,25 +75,34 @@ The hardest algorithm in the codebase. Converts per-file field presence into glo ### Bottom-up Merge -`merge()` at `discover/infer/paths.rs:241` propagates field presence up the tree via post-order traversal (`NodeEdge::End` = children processed before parent): +`merge()` at `discover/infer/paths.rs:241` propagates field presence up the tree +via post-order traversal (`NodeEdge::End` = children processed before parent): For each non-leaf node, aggregate children: -- `node.all` = intersection of all children's `all` sets (field present in every file under this subtree) + +- `node.all` = intersection of all children's `all` sets (field present in every + file under this subtree) - `node.any` = intersection of all children's `any` sets -After merge, each internal node knows which fields appear in all descendants vs. some descendants. +After merge, each internal node knows which fields appear in all descendants vs. +some descendants. ### Glob Collapsing -`infer_paths()` at `discover/infer/paths.rs:171` converts the tree into glob patterns in three phases: +`infer_paths()` at `discover/infer/paths.rs:171` converts the tree into glob +patterns in three phases: + +**Phase 1 — Seed from leaves.** For each leaf node, add every field in `any` as +a shallow glob (`dir/*`): -**Phase 1 — Seed from leaves.** For each leaf node, add every field in `any` as a shallow glob (`dir/*`): ``` GlobMap::insert_shallow(dir_path) → entries[dir] = Shallow ``` **Phase 2 — Collapse from parents.** Post-order walk over non-leaf nodes: -- Fields in `all` (present in every file under this subtree): collapse both `allowed` and `required` to recursive glob (`dir/**`) + +- Fields in `all` (present in every file under this subtree): collapse both + `allowed` and `required` to recursive glob (`dir/**`) - Fields in `any \ all` (some but not all files): collapse only `allowed` ``` @@ -84,40 +111,59 @@ GlobMap::collapse(ancestor_path): entries[ancestor_path] = Recursive // replace with ** ``` -**Phase 3 — Emit globs.** `GlobMap::to_globs()` converts entries to sorted strings: +**Phase 3 — Emit globs.** `GlobMap::to_globs()` converts entries to sorted +strings: + - `Shallow` → `dir/*` (or `*` for root) - `Recursive` → `dir/**` (or `**` for root) ### Example -Given files: `blog/a.md` (title, tags), `blog/b.md` (title), `notes/c.md` (title, tags): +Given files: `blog/a.md` (title, tags), `blog/b.md` (title), `notes/c.md` +(title, tags): - Leaf `blog/`: all={title}, any={title, tags} - Leaf `notes/`: all={title, tags}, any={title, tags} - After merge, root: all={title}, any={title, tags} - For `title`: root.all → collapse to `**` in both allowed and required -- For `tags`: root.any\all → collapse allowed to `**`; blog.any has tags with Shallow initially, notes.all has tags → required gets `notes/**` +- For `tags`: root.any\all → collapse allowed to `**`; blog.any has tags with + Shallow initially, notes.all has tags → required gets `notes/**` -Result: `title` allowed=`["**"]` required=`["**"]`; `tags` allowed=`["**"]` required=`["notes/**"]`. +Result: `title` allowed=`["**"]` required=`["**"]`; `tags` allowed=`["**"]` +required=`["notes/**"]`. ## Categorical Inference -`infer_constraints()` at `discover/infer/constraints/mod.rs:13` applies a heuristic after type+path inference: +`infer_constraints()` at `discover/infer/constraints/mod.rs:13` applies a +heuristic after type+path inference: -1. **Type check** — field must be String, Integer, Array(String), or Array(Integer) +1. **Type check** — field must be String, Integer, Array(String), or + Array(Integer) 2. **Distinct cap** — `distinct_values.len() <= max_categories` (default 10) -3. **Repetition** — `occurrence_count / distinct_values.len() >= min_repetition` (default 3) +3. **Repetition** — `occurrence_count / distinct_values.len() >= min_repetition` + (default 3) -Distinct values and occurrence counts are collected during type inference in `collect_distinct_values()` at `discover/infer/types.rs:86`. For arrays, counting is element-level (each array element is one occurrence). Null values are excluded. +Distinct values and occurrence counts are collected during type inference in +`collect_distinct_values()` at `discover/infer/types.rs:86`. For arrays, +counting is element-level (each array element is one occurrence). Null values +are excluded. -Values are converted from `serde_json::Value` to `toml::Value` (String→String, Number→Integer) and sorted for deterministic output. +Values are converted from `serde_json::Value` to `toml::Value` (String→String, +Number→Integer) and sorted for deterministic output. ## Orchestration -`InferredSchema::infer(scanned)` at `discover/infer/mod.rs:78` runs three phases sequentially: +`InferredSchema::infer(scanned)` at `discover/infer/mod.rs:78` runs three phases +sequentially: -1. **Type phase** — `infer_field_types(scanned)` → `BTreeMap` (widened types, file lists, distinct values, occurrence counts, nullable flags) -2. **Path phase** — `DirectoryTree::from(scanned)` → `tree.infer_paths()` → `BTreeMap` (allowed + required globs) -3. **Merge** — combine type info and path info into `Vec`, sorted by name +1. **Type phase** — `infer_field_types(scanned)` → + `BTreeMap` (widened types, file lists, distinct + values, occurrence counts, nullable flags) +2. **Path phase** — `DirectoryTree::from(scanned)` → `tree.infer_paths()` → + `BTreeMap` (allowed + required globs) +3. **Merge** — combine type info and path info into `Vec`, sorted + by name -The categorical heuristic is NOT applied here — it runs later in `from_inferred()` (`schema/config.rs`) or `update::run()` when converting `InferredField` to `TomlField`. +The categorical heuristic is NOT applied here — it runs later in +`from_inferred()` (`schema/config.rs`) or `update::run()` when converting +`InferredField` to `TomlField`. diff --git a/docs/spec/release.md b/docs/spec/release.md index 571b7dc..2747110 100644 --- a/docs/spec/release.md +++ b/docs/spec/release.md @@ -2,18 +2,22 @@ ## Overview -Releases are semi-automated. You decide when to release, tools handle the mechanics: +Releases are semi-automated. You decide when to release, tools handle the +mechanics: -1. **cocogitto (`cog bump`)** — bumps version, generates changelog, commits, tags, pushes +1. **cocogitto (`cog bump`)** — bumps version, generates changelog, commits, + tags, pushes 2. **cargo-dist** — builds cross-platform binaries, creates GitHub Release -No publishing to external registries (crates.io, Homebrew, npm) for now. `publish = false` in `Cargo.toml` blocks all registry publishing. +No publishing to external registries (crates.io, Homebrew, npm) for now. +`publish = false` in `Cargo.toml` blocks all registry publishing. ## Tools ### cocogitto (`cog bump`) -Automates the full release cycle: version bump → changelog generation → commit → tag → push. Determines the next version from conventional commits. +Automates the full release cycle: version bump → changelog generation → commit → +tag → push. Determines the next version from conventional commits. **Install:** `cargo install cocogitto` @@ -38,11 +42,13 @@ repository = "mdvs" - `post_bump_hooks` — auto-pushes the commit and tag after bumping - `[changelog]` — generates CHANGELOG.md with GitHub commit/PR links -See the [conventional commits spec](https://www.conventionalcommits.org/) for the syntax. +See the [conventional commits spec](https://www.conventionalcommits.org/) for +the syntax. ### cargo-dist -Builds cross-platform binaries and creates GitHub Releases. Triggered by tag pushes. +Builds cross-platform binaries and creates GitHub Releases. Triggered by tag +pushes. **Install:** `cargo install cargo-dist` @@ -70,9 +76,11 @@ ci = "github" - `[profile.dist]` — custom Cargo profile for release builds (LTO enabled) - `[package.metadata.dist]` — package-level config (targets, installers) -- `[workspace.metadata.dist]` — workspace-level config (cargo-dist version, CI backend) +- `[workspace.metadata.dist]` — workspace-level config (cargo-dist version, CI + backend) -**Regenerating the workflow:** if you change the dist config, run `dist generate` to update `.github/workflows/release.yml`. +**Regenerating the workflow:** if you change the dist config, run +`dist generate` to update `.github/workflows/release.yml`. ## Pipelines @@ -81,16 +89,20 @@ ci = "github" **Triggers:** push to any branch, any PR to `main` **Steps:** + 1. Checkout with full history (`fetch-depth: 0`) 2. `cocogitto-action@v4.1.0` — `cog check --from-latest-tag` -Validates that all commits since the latest tag follow conventional commit format. Runs on all branches so non-conventional commits are caught immediately, not just at merge time. +Validates that all commits since the latest tag follow conventional commit +format. Runs on all branches so non-conventional commits are caught immediately, +not just at merge time. ### CI (`.github/workflows/ci.yml`) **Triggers:** push to `main`, any PR to `main` **Steps:** + 1. `cargo build` 2. `cargo test` 3. `cargo clippy -- -D warnings` @@ -100,12 +112,15 @@ Validates that all commits since the latest tag follow conventional commit forma ### Release (`.github/workflows/release.yml`) -**Triggers:** push of a tag matching `*[0-9]+.[0-9]+.[0-9]+*` (e.g., `v0.1.0`, `v0.1.0-rc.1`), and PRs (dry-run only) +**Triggers:** push of a tag matching `*[0-9]+.[0-9]+.[0-9]+*` (e.g., `v0.1.0`, +`v0.1.0-rc.1`), and PRs (dry-run only) **Jobs:** -1. **plan** (ubuntu) — runs `dist host --steps=create` to determine what to build, outputs a manifest -2. **build-local-artifacts** (4 runners in parallel) — compiles platform-specific binaries: +1. **plan** (ubuntu) — runs `dist host --steps=create` to determine what to + build, outputs a manifest +2. **build-local-artifacts** (4 runners in parallel) — compiles + platform-specific binaries: - `macos-14` → `mdvs-aarch64-apple-darwin.tar.xz` - `macos-13` → `mdvs-x86_64-apple-darwin.tar.xz` - `ubuntu-22.04` → `mdvs-x86_64-unknown-linux-gnu.tar.xz` @@ -120,7 +135,8 @@ Validates that all commits since the latest tag follow conventional commit forma On PRs, only the **plan** job runs (dry-run validation, no builds or uploads). -Prerelease tags (e.g., `v0.1.0-rc.1`) create a GitHub Release marked as prerelease. +Prerelease tags (e.g., `v0.1.0-rc.1`) create a GitHub Release marked as +prerelease. ## Making a Release @@ -130,19 +146,21 @@ Prerelease tags (e.g., `v0.1.0-rc.1`) create a GitHub Release marked as prerelea cog bump --dry-run --patch # or --minor, --major, --auto ``` -Shows what the next version would be without doing anything. Review with the user before proceeding. +Shows what the next version would be without doing anything. Review with the +user before proceeding. ### Release commands -| Command | Example | Use case | -|---------|---------|----------| -| `cog bump --patch` | 0.1.0 → 0.1.1 | Bug fixes | -| `cog bump --minor` | 0.1.0 → 0.2.0 | New features | -| `cog bump --major` | 0.1.0 → 1.0.0 | Breaking changes | -| `cog bump --auto` | (auto-detected) | Let commit types decide the level | -| `cog bump --minor --pre rc` | 0.1.0 → 0.1.0-rc | Prerelease / testing | +| Command | Example | Use case | +| --------------------------- | ---------------- | --------------------------------- | +| `cog bump --patch` | 0.1.0 → 0.1.1 | Bug fixes | +| `cog bump --minor` | 0.1.0 → 0.2.0 | New features | +| `cog bump --major` | 0.1.0 → 1.0.0 | Breaking changes | +| `cog bump --auto` | (auto-detected) | Let commit types decide the level | +| `cog bump --minor --pre rc` | 0.1.0 → 0.1.0-rc | Prerelease / testing | Each command: + 1. Bumps version in `Cargo.toml` 2. Generates/updates `CHANGELOG.md` from conventional commits 3. Creates a bump commit @@ -154,6 +172,7 @@ The tag push triggers the Release pipeline automatically. ### `--auto` behavior `--auto` reads commits since the last tag and picks the bump level: + - Any `feat:` → minor - Only `fix:`, `refactor:`, etc. → patch - Any `BREAKING CHANGE` → major @@ -169,11 +188,13 @@ gh run watch # live follow (optional) ``` If a build fails: + ```bash gh run view --job= --log-failed # see failure logs ``` After success, verify the GitHub Release: + ```bash gh release view v ``` @@ -186,23 +207,28 @@ dist plan # shows what artifacts would be built ## Changelog -`CHANGELOG.md` is auto-generated by `cog bump` from conventional commits. The `[changelog]` section in `cog.toml` configures the output format, including GitHub links to commits and PRs. +`CHANGELOG.md` is auto-generated by `cog bump` from conventional commits. The +`[changelog]` section in `cog.toml` configures the output format, including +GitHub links to commits and PRs. -The changelog groups entries by commit type (Features, Bug Fixes, etc.) and includes the full list of changes since the previous tag. +The changelog groups entries by commit type (Features, Bug Fixes, etc.) and +includes the full list of changes since the previous tag. ## Configuration -| File | Purpose | -|------|---------| -| `cog.toml` | cocogitto config — tag prefix, bump hooks, changelog, commit validation | -| `Cargo.toml` | Version field, `[profile.dist]`, `[package.metadata.dist]`, `[workspace.metadata.dist]` | -| `.github/workflows/release.yml` | Auto-generated by `dist generate`, do not hand-edit | +| File | Purpose | +| ------------------------------- | --------------------------------------------------------------------------------------- | +| `cog.toml` | cocogitto config — tag prefix, bump hooks, changelog, commit validation | +| `Cargo.toml` | Version field, `[profile.dist]`, `[package.metadata.dist]`, `[workspace.metadata.dist]` | +| `.github/workflows/release.yml` | Auto-generated by `dist generate`, do not hand-edit | ## Important notes - `publish = false` — no crates.io publishing. Releases are GitHub-only for now. -- Tag format is `v{version}` (e.g., `v0.1.0`), configured by `tag_prefix = "v"` in `cog.toml`. -- Prerelease tags (e.g., `v0.1.0-rc.1`) create a GitHub Release marked as prerelease. +- Tag format is `v{version}` (e.g., `v0.1.0`), configured by `tag_prefix = "v"` + in `cog.toml`. +- Prerelease tags (e.g., `v0.1.0-rc.1`) create a GitHub Release marked as + prerelease. - If cargo-dist config changes, regenerate the workflow: `dist generate` ## Future: Publishing to Registries @@ -210,7 +236,8 @@ The changelog groups entries by commit type (Features, Bug Fixes, etc.) and incl When ready to publish externally: 1. **crates.io** — remove `publish = false` from `Cargo.toml` -2. **Homebrew** — add `"homebrew"` to `installers` and `publish-jobs` in `Cargo.toml`, create `edochi/homebrew-tap` repo +2. **Homebrew** — add `"homebrew"` to `installers` and `publish-jobs` in + `Cargo.toml`, create `edochi/homebrew-tap` repo 3. **npm** — add `"npm"` to `installers` and `publish-jobs` in `Cargo.toml` Then run `dist generate` to update the release workflow. diff --git a/docs/spec/scaffolding.md b/docs/spec/scaffolding.md index 69cf632..1908c89 100644 --- a/docs/spec/scaffolding.md +++ b/docs/spec/scaffolding.md @@ -1,22 +1,32 @@ # Scaffolding -How mdvs integrates with agent harnesses (Claude Code, Codex, Cursor, OpenCode, Antigravity) and how to add a new platform. +How mdvs integrates with agent harnesses (Claude Code, Codex, Cursor, OpenCode, +Antigravity) and how to add a new platform. -This page documents the internal architecture. User-facing instructions live in [book/src/recipes/agentic-harnesses-and-agentic-ides.md](../../book/src/recipes/agentic-harnesses-and-agentic-ides.md). +This page documents the internal architecture. User-facing instructions live in +[book/src/recipes/agentic-harnesses-and-agentic-ides.md](../../book/src/recipes/agentic-harnesses-and-agentic-ides.md). ## Overview -Three install-time commands write content to disk in the harness's expected location: +Three install-time commands write content to disk in the harness's expected +location: -- `mdvs scaffold skill [--platform ]` — emits a comprehensive `SKILL.md` (Agent Skills standard). -- `mdvs scaffold snippet [--platform ]` — emits a project-rules block to paste into `AGENTS.md` / `CLAUDE.md` / `.cursor/rules/`. -- `mdvs scaffold hook --platform ` — emits a per-platform PostToolUse hook config (JSON snippet) that the harness's settings file ingests. +- `mdvs scaffold skill [--platform ]` — emits a comprehensive `SKILL.md` + (Agent Skills standard). +- `mdvs scaffold snippet [--platform ]` — emits a project-rules block to + paste into `AGENTS.md` / `CLAUDE.md` / `.cursor/rules/`. +- `mdvs scaffold hook --platform ` — emits a per-platform PostToolUse hook + config (JSON snippet) that the harness's settings file ingests. One runtime command handles every hook invocation: -- `mdvs hook handle --platform --kind ` — reads the harness's stdin JSON, walks up to find `mdvs.toml`, runs the requested logic, writes a platform-shaped envelope to stdout, exits 0. +- `mdvs hook handle --platform --kind ` — reads + the harness's stdin JSON, walks up to find `mdvs.toml`, runs the requested + logic, writes a platform-shaped envelope to stdout, exits 0. -All four commands share the same per-platform data: `scaffolding/platforms//platform.toml`. Adding a new harness is adding one toml file; no Rust changes. +All four commands share the same per-platform data: +`scaffolding/platforms//platform.toml`. Adding a new harness is adding one +toml file; no Rust changes. ## Directory layout (`crates/mdvs/scaffolding/`) @@ -34,7 +44,9 @@ scaffolding/ └── antigravity/platform.toml — skill + snippet only, no [hooks] ``` -The entire `scaffolding/` tree is bundled into the binary at build time via `include_dir!`. No disk access at runtime; users get a single binary that knows about every platform. +The entire `scaffolding/` tree is bundled into the binary at build time via +`include_dir!`. No disk access at runtime; users get a single binary that knows +about every platform. ## `platform.toml` schema @@ -66,29 +78,44 @@ template = """ # JSON, parsed at load time, with <> placehol """ ``` -The `[hooks]` section is what makes a platform "hook-capable." Platforms without it (OpenCode, Antigravity) get full skill + snippet support, but `mdvs scaffold hook --platform ` and `mdvs hook handle --platform ` both refuse with a pointer at the recipe page. +The `[hooks]` section is what makes a platform "hook-capable." Platforms without +it (OpenCode, Antigravity) get full skill + snippet support, but +`mdvs scaffold hook --platform ` and `mdvs hook handle --platform ` +both refuse with a pointer at the recipe page. ## Template substitution (`scaffold/template.rs`) -`platform.toml` declares the JSON shapes the harness expects. Variable parts (the agent message, the user message, the install-time command strings) are filled in at the point of use via `<>` placeholders. +`platform.toml` declares the JSON shapes the harness expects. Variable parts +(the agent message, the user message, the install-time command strings) are +filled in at the point of use via `<>` placeholders. ### Marker syntax -A marker is a JSON string whose **entire content** matches `<>` where `NAME` is an uppercase identifier. The full-string-match rule means `"hello <>"` is **NOT** a marker — it's a literal string with text that happens to contain angle brackets. This avoids the escaping and partial-substitution edge cases of `printf`-style templating. +A marker is a JSON string whose **entire content** matches `<>` where +`NAME` is an uppercase identifier. The full-string-match rule means +`"hello <>"` is **NOT** a marker — it's a literal string with text that +happens to contain angle brackets. This avoids the escaping and +partial-substitution edge cases of `printf`-style templating. ### Substitution rules Given `vars: HashMap<&str, Option>`: -| `vars[NAME]` | Behavior | -|---|---| -| `Some(value)` | Replace the marker string with the value as a JSON string node. | -| `None` | **Prune**: remove the parent key (in objects) or array element. | -| Not present in vars | Same as `None`. | +| `vars[NAME]` | Behavior | +| ------------------- | --------------------------------------------------------------- | +| `Some(value)` | Replace the marker string with the value as a JSON string node. | +| `None` | **Prune**: remove the parent key (in objects) or array element. | +| Not present in vars | Same as `None`. | -The prune-on-`None` rule handles per-kind variance cleanly. Example: Claude Code's envelope template has both `<>` and `<>`. For `validate`, both are populated → full envelope. For `search-nudge`, only `<>` is populated → the `<>` marker is pruned, taking `"systemMessage"` with it. The agent sees the tip in `additionalContext`; the user UI stays quiet. +The prune-on-`None` rule handles per-kind variance cleanly. Example: Claude +Code's envelope template has both `<>` and `<>`. For `validate`, +both are populated → full envelope. For `search-nudge`, only `<>` is +populated → the `<>` marker is pruned, taking `"systemMessage"` with +it. The agent sees the tip in `additionalContext`; the user UI stays quiet. -Cursor's envelope template doesn't reference `<>` at all (Cursor's `postToolUse` has no user channel). Whether mdvs passes `USER_MSG=Some(...)` or `None` makes no difference — same output either way. +Cursor's envelope template doesn't reference `<>` at all (Cursor's +`postToolUse` has no user channel). Whether mdvs passes `USER_MSG=Some(...)` or +`None` makes no difference — same output either way. ### Implementation @@ -96,7 +123,10 @@ Cursor's envelope template doesn't reference `<>` at all (Cursor's `po pub fn substitute(template: &Value, vars: &HashMap<&str, Option>) -> Value; ``` -Walks the parsed `serde_json::Value` tree, returning a new tree with markers replaced and pruned keys/elements removed. No string concatenation, no escaping required from the template author — substitution operates on JSON nodes, not raw text. +Walks the parsed `serde_json::Value` tree, returning a new tree with markers +replaced and pruned keys/elements removed. No string concatenation, no escaping +required from the template author — substitution operates on JSON nodes, not raw +text. Tests covering all the rules: `scaffold::template::tests` (14 tests). @@ -110,12 +140,17 @@ $ mdvs scaffold hook --platform claude-code Loads `Platform`, builds two vars: -- `<>` → `"mdvs hook handle --platform claude-code --kind validate"` -- `<>` → `"mdvs hook handle --platform claude-code --kind search-nudge"` +- `<>` → + `"mdvs hook handle --platform claude-code --kind validate"` +- `<>` → + `"mdvs hook handle --platform claude-code --kind search-nudge"` -Substitutes into `hooks.config` template. Injects a `_comment` field at the top of the resulting object explaining where to merge it. Emits to stdout (clean for piping); install-path hint goes to stderr. +Substitutes into `hooks.config` template. Injects a `_comment` field at the top +of the resulting object explaining where to merge it. Emits to stdout (clean for +piping); install-path hint goes to stderr. -For platforms without `[hooks]`: refuses with a pointer that still directs users to `mdvs scaffold skill|snippet`. +For platforms without `[hooks]`: refuses with a pointer that still directs users +to `mdvs scaffold skill|snippet`. ### `mdvs hook handle` (runtime) @@ -126,26 +161,37 @@ $ echo '{"tool_input":{"file_path":"kb/note.md"},"cwd":"/path"}' \ For `--kind validate`: -1. Read stdin JSON, extract `tool_input.file_path`. Silent if absent or not `.md`. -2. Walk up from the file's directory looking for `mdvs.toml`. Silent if no vault. +1. Read stdin JSON, extract `tool_input.file_path`. Silent if absent or not + `.md`. +2. Walk up from the file's directory looking for `mdvs.toml`. Silent if no + vault. 3. Run `cmd::check::run` on the vault directly (no subprocess; no shell). 4. If no violations and no error → silent exit 0. -5. Render the result twice: `--output markdown` for the agent channel (uncapped), `--output pretty` for the user channel (capped at `MAX_USER_LINES = 15` with a `...` truncation marker). +5. Render the result twice: `--output markdown` for the agent channel + (uncapped), `--output pretty` for the user channel (capped at + `MAX_USER_LINES = 15` with a `...` truncation marker). 6. Append a skill pointer to the agent markdown. -7. Build vars: `<>` = agent markdown, `<>` = pretty (`Some` for validate; `None` would prune). +7. Build vars: `<>` = agent markdown, `<>` = pretty (`Some` for + validate; `None` would prune). 8. Substitute into `hooks.envelope`. Emit to stdout, exit 0. For `--kind search-nudge`: 1. Read stdin JSON, extract `cwd` (or `pwd` fallback). 2. Walk up to find `mdvs.toml`. Silent if no vault. -3. Match `tool_input.command` against search-tool patterns (`grep`, `rg `, `ripgrep`, `find `, `fd `, `fdfind `, `ag `, `ack `, `git grep`). -4. Build vars: `<>` = the static tip, `<>` = `None` (search-nudge stays out of the user UI). +3. Match `tool_input.command` against search-tool patterns (`grep`, `rg `, + `ripgrep`, `find `, `fd `, `fdfind `, `ag `, `ack `, `git grep`). +4. Build vars: `<>` = the static tip, `<>` = `None` (search-nudge + stays out of the user UI). 5. Substitute into `hooks.envelope`. Emit to stdout, exit 0. -The hook **never blocks** (always exits 0). Violations and tips surface to the agent through `additionalContext`-style channels; the agent decides what to do next. The "warning, not block" design intentionally keeps mdvs out of the harness's permission flow — the schema is meant to evolve with the KB. +The hook **never blocks** (always exits 0). Violations and tips surface to the +agent through `additionalContext`-style channels; the agent decides what to do +next. The "warning, not block" design intentionally keeps mdvs out of the +harness's permission flow — the schema is meant to evolve with the KB. -For platforms without `[hooks]`: refuses with the same pointer as `mdvs scaffold hook`. +For platforms without `[hooks]`: refuses with the same pointer as +`mdvs scaffold hook`. ## Adding a new platform @@ -153,46 +199,69 @@ Three categories, ordered by ease. ### A) New harness with a Claude-Code-style hook config -Examples of what would qualify: a hypothetical harness that uses the same `{"hooks": {"": [{"matcher": "...", "hooks": [{"type": "command", "command": "..."}]}]}}` nesting but at a different config path or with a different event-name capitalization. +Examples of what would qualify: a hypothetical harness that uses the same +`{"hooks": {"": [{"matcher": "...", "hooks": [{"type": "command", "command": "..."}]}]}}` +nesting but at a different config path or with a different event-name +capitalization. Steps: -1. Create `scaffolding/platforms//platform.toml` with the right `[meta]`, `[skill]`, `[snippet]`, `[hooks]` sections. -2. Copy `claude-code/platform.toml`'s `[hooks.envelope]` and `[hooks.config]` templates as a starting point; adjust event names and matcher strings to match the harness's docs. -3. `cargo test --features testing-mocks scaffold::platform` — verifies the toml parses cleanly and the template is valid JSON. Add a per-platform test if the new shape has distinctive landmarks. +1. Create `scaffolding/platforms//platform.toml` with the right `[meta]`, + `[skill]`, `[snippet]`, `[hooks]` sections. +2. Copy `claude-code/platform.toml`'s `[hooks.envelope]` and `[hooks.config]` + templates as a starting point; adjust event names and matcher strings to + match the harness's docs. +3. `cargo test --features testing-mocks scaffold::platform` — verifies the toml + parses cleanly and the template is valid JSON. Add a per-platform test if the + new shape has distinctive landmarks. No Rust code changes needed. ### B) New harness with a completely different JSON shape -Cursor was the original test case: snake-case `additional_context` at the top level, no `hookSpecificOutput` wrapper, no `systemMessage`-equivalent, flat matcher entries on the config side, top-level `"version": 1`. +Cursor was the original test case: snake-case `additional_context` at the top +level, no `hookSpecificOutput` wrapper, no `systemMessage`-equivalent, flat +matcher entries on the config side, top-level `"version": 1`. -Same three steps as (A), but the templates are written from scratch to match the harness's actual schema. The `<>` and `<>` markers can appear anywhere in the envelope template (or not at all, if the harness has no equivalent channel). The `<>` and `<>` markers are required somewhere in the config template (otherwise `mdvs scaffold hook` would emit an unfilled snippet). +Same three steps as (A), but the templates are written from scratch to match the +harness's actual schema. The `<>` and `<>` markers can appear +anywhere in the envelope template (or not at all, if the harness has no +equivalent channel). The `<>` and `<>` markers +are required somewhere in the config template (otherwise `mdvs scaffold hook` +would emit an unfilled snippet). Still no Rust code changes. ### C) New harness with a non-shell hook surface -OpenCode (TypeScript plugin API) and Antigravity (undocumented post-rebrand) are current examples. There's no JSON-config + shell-command pathway for mdvs to slot into. +OpenCode (TypeScript plugin API) and Antigravity (undocumented post-rebrand) are +current examples. There's no JSON-config + shell-command pathway for mdvs to +slot into. Steps: -1. Create `scaffolding/platforms//platform.toml` with `[meta]`, `[skill]`, `[snippet]` only — omit `[hooks]` entirely. -2. `mdvs scaffold skill` and `mdvs scaffold snippet` will work; `mdvs scaffold hook --platform ` and `mdvs hook handle --platform ` will refuse with a helpful pointer. -3. If/when the harness adds a documented shell-command hook surface, fill in `[hooks]` to upgrade to category (A) or (B). +1. Create `scaffolding/platforms//platform.toml` with `[meta]`, `[skill]`, + `[snippet]` only — omit `[hooks]` entirely. +2. `mdvs scaffold skill` and `mdvs scaffold snippet` will work; + `mdvs scaffold hook --platform ` and + `mdvs hook handle --platform ` will refuse with a helpful pointer. +3. If/when the harness adds a documented shell-command hook surface, fill in + `[hooks]` to upgrade to category (A) or (B). ### What `<>` names are available -Hard-coded set, defined in `cmd/hook/handle.rs::build_envelope` (envelope) and `cmd/scaffold/hook.rs::build_config` (config): +Hard-coded set, defined in `cmd/hook/handle.rs::build_envelope` (envelope) and +`cmd/scaffold/hook.rs::build_config` (config): -| Marker | Available in | Source | -|---|---|---| -| `<>` | envelope | agent-context message (markdown violation report + skill pointer for validate; static tip for search-nudge) | -| `<>` | envelope | user-visible message (pretty render of violations, capped). `None` for search-nudge (causes prune). | -| `<>` | config | `"mdvs hook handle --platform --kind validate"` | -| `<>` | config | `"mdvs hook handle --platform --kind search-nudge"` | +| Marker | Available in | Source | +| ---------------------- | ------------ | ----------------------------------------------------------------------------------------------------------- | +| `<>` | envelope | agent-context message (markdown violation report + skill pointer for validate; static tip for search-nudge) | +| `<>` | envelope | user-visible message (pretty render of violations, capped). `None` for search-nudge (causes prune). | +| `<>` | config | `"mdvs hook handle --platform --kind validate"` | +| `<>` | config | `"mdvs hook handle --platform --kind search-nudge"` | -Adding a new marker requires a small Rust change in the consumer that fills it; the template engine itself doesn't know about marker names. +Adding a new marker requires a small Rust change in the consumer that fills it; +the template engine itself doesn't know about marker names. ## Rust shape (`crates/mdvs/src/scaffold/`) @@ -203,15 +272,33 @@ scaffold/ └── template.rs — substitute() ``` -`Platform` is a plain struct loaded from `platform.toml`. No enum (platforms aren't a fixed set), no `dyn Trait` (no library-style downstream extension; this is a binary). New harnesses come from new toml files, never from new Rust types. The project's "enum dispatch, no `dyn Trait`" rule (from [architecture.md](architecture.md#enum-dispatch-pattern)) still applies to concerns where finiteness matters (backends, embedders, value stages); platforms aren't one of those. +`Platform` is a plain struct loaded from `platform.toml`. No enum (platforms +aren't a fixed set), no `dyn Trait` (no library-style downstream extension; this +is a binary). New harnesses come from new toml files, never from new Rust types. +The project's "enum dispatch, no `dyn Trait`" rule (from +[architecture.md](architecture.md#enum-dispatch-pattern)) still applies to +concerns where finiteness matters (backends, embedders, value stages); platforms +aren't one of those. ## Why `mdvs hook handle` lives in Rust and not in shell scripts -mdvs is already a cross-platform Rust binary. The shell-script approach we shipped initially (six `.sh` files across three platforms) only worked on POSIX systems and required `jq` on `PATH` — a real blocker for Windows users and a friction point even on Mac/Linux. Pulling the logic into a single mdvs subcommand: +mdvs is already a cross-platform Rust binary. The shell-script approach we +shipped initially (six `.sh` files across three platforms) only worked on POSIX +systems and required `jq` on `PATH` — a real blocker for Windows users and a +friction point even on Mac/Linux. Pulling the logic into a single mdvs +subcommand: -- Eliminates the per-OS implementation matrix; one Rust build target serves every OS mdvs already supports. +- Eliminates the per-OS implementation matrix; one Rust build target serves + every OS mdvs already supports. - Drops the `jq` runtime dependency. -- Lets the install-time `command:` field in the harness's settings file be a one-liner pointing at `mdvs hook handle` — no separate script files to install, no shell-escaping concerns. -- Centralizes the hook contract in one testable place. Bug fixes ship via a normal mdvs release rather than per-user shell-script edits. - -The trade-off is that the per-harness JSON shape now lives inside the binary (via the embedded `platform.toml` files) instead of in editable shell scripts. The template-driven design above mitigates this: users (and contributors) can override by adding a new `platform.toml` to the source tree, no Rust touch required. +- Lets the install-time `command:` field in the harness's settings file be a + one-liner pointing at `mdvs hook handle` — no separate script files to + install, no shell-escaping concerns. +- Centralizes the hook contract in one testable place. Bug fixes ship via a + normal mdvs release rather than per-user shell-script edits. + +The trade-off is that the per-harness JSON shape now lives inside the binary +(via the embedded `platform.toml` files) instead of in editable shell scripts. +The template-driven design above mitigates this: users (and contributors) can +override by adding a new `platform.toml` to the source tree, no Rust touch +required. diff --git a/docs/spec/search.md b/docs/spec/search.md index 38525f8..4a90d49 100644 --- a/docs/spec/search.md +++ b/docs/spec/search.md @@ -1,61 +1,100 @@ # Search -Deep-dive into the search pipeline. For the module map see [architecture.md](./architecture.md). +Deep-dive into the search pipeline. For the module map see +[architecture.md](./architecture.md). -Search delegates to LanceDB. mdvs's role is: translate the query and the optional `--where` clause into a LanceDB query, dispatch on `SearchMode`, deduplicate to the best chunk per file. Key files: `search.rs` (mode enum, score-column resolution), `index/backend/` (post-TODO-0179 split: `mod.rs` holds the `Backend` enum + `SearchHit`; `search.rs` holds `LanceBackend::search` and the `--where` translator). +Search delegates to LanceDB. mdvs's role is: translate the query and the +optional `--where` clause into a LanceDB query, dispatch on `SearchMode`, +deduplicate to the best chunk per file. Key files: `search.rs` (mode enum, +score-column resolution), `index/backend/` (post-TODO-0179 split: `mod.rs` holds +the `Backend` enum + `SearchHit`; `search.rs` holds `LanceBackend::search` and +the `--where` translator). ## SearchMode `SearchMode` (`search.rs`): -| Variant | Score column on result rows | What runs | -|---|---|---| -| `Semantic` | `_distance` (mapped to `1 - d`) | `.nearest_to(query_embedding).distance_type(Cosine)` | -| `Fulltext` | `_score` | `.full_text_search(FullTextSearchQuery::new(query))` | -| `Hybrid` (default) | `_relevance_score` | both of the above + `.rerank(RrfReranker::default())` | +| Variant | Score column on result rows | What runs | +| ------------------ | ------------------------------- | ----------------------------------------------------- | +| `Semantic` | `_distance` (mapped to `1 - d`) | `.nearest_to(query_embedding).distance_type(Cosine)` | +| `Fulltext` | `_score` | `.full_text_search(FullTextSearchQuery::new(query))` | +| `Hybrid` (default) | `_relevance_score` | both of the above + `.rerank(RrfReranker::default())` | -The CLI flag `--mode {semantic,fulltext,hybrid}` selects the variant; the default is `Hybrid`. `Semantic` and `Hybrid` require the embedding model to be loaded; `Fulltext` does not. +The CLI flag `--mode {semantic,fulltext,hybrid}` selects the variant; the +default is `Hybrid`. `Semantic` and `Hybrid` require the embedding model to be +loaded; `Fulltext` does not. ## --where translation -`translate_where_to_struct` (`index/backend/search.rs`) rewrites the user clause so that: - -- Bare frontmatter field names get a `data.` prefix (so `status = 'active'` becomes `data.status = 'active'`). -- Identifiers immediately followed by `(` are treated as **function calls**, not field names — `lower(status)` is rewritten to `lower(data.status)`, not `data.lower(...)`. -- Internal columns (`chunk_text`, `start_line`, `end_line`, `embedding`, …) are left bare. -- **References to `Array(Float)` field names produce an early error** with a clear message, before LanceDB sees the clause. This is the TODO-0159 mitigation — see [lancedb#3446](https://github.com/lancedb/lancedb/issues/3446) for the upstream report and [TODO-0159](todos/TODO-0159.md) for the local resolution notes. -- Date and timestamp literal keywords (`DATE '...'`, `TIMESTAMP '...'`) are protected from prefix injection by a literal-aware tokenizer. - -The translator is schema-aware: it loads the `data` Struct's child names + types from the Lance table schema once per `search()` call, via `float_list_child_names(schema)` for the Array(Float) guard and the full child-name set for prefixing. +`translate_where_to_struct` (`index/backend/search.rs`) rewrites the user clause +so that: + +- Bare frontmatter field names get a `data.` prefix (so `status = 'active'` + becomes `data.status = 'active'`). +- Identifiers immediately followed by `(` are treated as **function calls**, not + field names — `lower(status)` is rewritten to `lower(data.status)`, not + `data.lower(...)`. +- Internal columns (`chunk_text`, `start_line`, `end_line`, `embedding`, …) are + left bare. +- **References to `Array(Float)` field names produce an early error** with a + clear message, before LanceDB sees the clause. This is the TODO-0159 + mitigation — see + [lancedb#3446](https://github.com/lancedb/lancedb/issues/3446) for the + upstream report and [TODO-0159](todos/TODO-0159.md) for the local resolution + notes. +- Date and timestamp literal keywords (`DATE '...'`, `TIMESTAMP '...'`) are + protected from prefix injection by a literal-aware tokenizer. + +The translator is schema-aware: it loads the `data` Struct's child names + types +from the Lance table schema once per `search()` call, via +`float_list_child_names(schema)` for the Array(Float) guard and the full +child-name set for prefixing. ## Query execution (`LanceBackend::search`) `index/backend/search.rs::LanceBackend::search(mode, query, query_embedding, where_clause, limit)`: 1. **Open the table** — `conn.open_table("index")`. -2. **Build the query** — `table.query()` plus the mode-specific clauses listed in the `SearchMode` table above. +2. **Build the query** — `table.query()` plus the mode-specific clauses listed + in the `SearchMode` table above. 3. **Filter** — `.only_if()` if `where_clause` is `Some`. -4. **Over-fetch** — `.limit(limit.saturating_mul(OVER_FETCH_FACTOR))` with `OVER_FETCH_FACTOR = 3`. This compensates for chunks that will be dropped by the best-chunk-per-file dedupe step. -5. **Stream** — `.execute().await?` yields a `RecordBatchStream`; `try_collect()` materialises all batches. -6. **Limit-zero short circuit** — if the caller asked for `limit == 0` we return `Ok(vec![])` before reaching LanceDB, so the user sees no results instead of a cryptic "k must be positive" error. +4. **Over-fetch** — `.limit(limit.saturating_mul(OVER_FETCH_FACTOR))` with + `OVER_FETCH_FACTOR = 3`. This compensates for chunks that will be dropped by + the best-chunk-per-file dedupe step. +5. **Stream** — `.execute().await?` yields a `RecordBatchStream`; + `try_collect()` materialises all batches. +6. **Limit-zero short circuit** — if the caller asked for `limit == 0` we return + `Ok(vec![])` before reaching LanceDB, so the user sees no results instead of + a cryptic "k must be positive" error. ## Score column resolution -Each mode produces a different score column on the result rows. `resolve_score_column(mode)` (`search.rs`) returns the constant column name; for `Semantic` the raw value is `_distance` (smaller = closer), which the result-reader maps to `1.0 - d` so callers see "higher is better" uniformly across modes. +Each mode produces a different score column on the result rows. +`resolve_score_column(mode)` (`search.rs`) returns the constant column name; for +`Semantic` the raw value is `_distance` (smaller = closer), which the +result-reader maps to `1.0 - d` so callers see "higher is better" uniformly +across modes. ## Best-chunk-per-file dedupe Results come back per-chunk. mdvs collapses them in Rust: -1. Iterate the streamed rows in their LanceDB-returned order (already ranked by the mode's score). -2. Insert into a `HashMap`, keeping the highest-scored chunk per file. +1. Iterate the streamed rows in their LanceDB-returned order (already ranked by + the mode's score). +2. Insert into a `HashMap`, keeping the highest-scored chunk + per file. 3. Sort the resulting hits by score descending and truncate to `--limit`. -The over-fetch factor (×3) ensures that even when many of the top-ranked chunks come from a single file, we still have enough candidates from other files to fill the requested limit. +The over-fetch factor (×3) ensures that even when many of the top-ranked chunks +come from a single file, we still have enough candidates from other files to +fill the requested limit. ## Verbose snippet -In verbose mode `cmd/search.rs` reads the best chunk's text directly from the `chunk_text` column on the winning row. No second file read is needed — `chunk_text` is persisted on the index for exactly this purpose (and for the FTS index to operate on). +In verbose mode `cmd/search.rs` reads the best chunk's text directly from the +`chunk_text` column on the winning row. No second file read is needed — +`chunk_text` is persisted on the index for exactly this purpose (and for the FTS +index to operate on). ## Result Assembly @@ -71,17 +110,35 @@ pub struct SearchHit { } ``` -Assembled by downcasting LanceDB-returned Arrow arrays: `StringArray` for `filepath`/`chunk_text`, `Float64Array` for the score column, `Int32Array` for line ranges. +Assembled by downcasting LanceDB-returned Arrow arrays: `StringArray` for +`filepath`/`chunk_text`, `Float64Array` for the score column, `Int32Array` for +line ranges. ## Collision Avoidance -Internal columns (`chunk_id`, `file_id`, `chunk_index`, `start_line`, `end_line`, `chunk_text`, `embedding`, `filepath`, `content_hash`, `built_at`) live at the top level of the schema. Frontmatter fields live under the `data` Struct. The `--where` translator rewrites bare frontmatter names to `data.`, so the *qualified* SQL paths never clash even when a frontmatter field shares its name with an internal column. - -The user-visible question is what a bare reference in `--where` *means*. By default `filepath` in a `--where` clause refers to the internal column. If the user has a frontmatter field also called `filepath`, that's a collision the user has to resolve — the translator detects it and bails with an actionable error. - -Resolution at the translator layer uses `[search].internal_prefix` and `[search.aliases]`: - -- **Prefix** — `internal_prefix = "_"` renames the *bare reference* for all internal columns: the user writes `_filepath` to refer to the internal column, and `filepath` stays bare for the frontmatter field (translated to `data.filepath`). -- **Alias** — `[search.aliases].filepath = "path"` renames the *bare reference* for one internal column: the user writes `path` for the internal column and `filepath` stays bare for the frontmatter field. - -These only affect `--where` translation; the actual on-disk column names are always the literal constants from `index/storage.rs`. The translator handles the mapping in `translate_where_to_struct` (`index/backend/search.rs`). +Internal columns (`chunk_id`, `file_id`, `chunk_index`, `start_line`, +`end_line`, `chunk_text`, `embedding`, `filepath`, `content_hash`, `built_at`) +live at the top level of the schema. Frontmatter fields live under the `data` +Struct. The `--where` translator rewrites bare frontmatter names to +`data.`, so the _qualified_ SQL paths never clash even when a frontmatter +field shares its name with an internal column. + +The user-visible question is what a bare reference in `--where` _means_. By +default `filepath` in a `--where` clause refers to the internal column. If the +user has a frontmatter field also called `filepath`, that's a collision the user +has to resolve — the translator detects it and bails with an actionable error. + +Resolution at the translator layer uses `[search].internal_prefix` and +`[search.aliases]`: + +- **Prefix** — `internal_prefix = "_"` renames the _bare reference_ for all + internal columns: the user writes `_filepath` to refer to the internal column, + and `filepath` stays bare for the frontmatter field (translated to + `data.filepath`). +- **Alias** — `[search.aliases].filepath = "path"` renames the _bare reference_ + for one internal column: the user writes `path` for the internal column and + `filepath` stays bare for the frontmatter field. + +These only affect `--where` translation; the actual on-disk column names are +always the literal constants from `index/storage.rs`. The translator handles the +mapping in `translate_where_to_struct` (`index/backend/search.rs`). diff --git a/docs/spec/shared.md b/docs/spec/shared.md index ad0bda6..c51f085 100644 --- a/docs/spec/shared.md +++ b/docs/spec/shared.md @@ -1,6 +1,7 @@ # Shared Types -Output and validation types used across commands. All defined in `src/output.rs` unless noted. +Output and validation types used across commands. All defined in `src/output.rs` +unless noted. ## Output Format @@ -8,7 +9,8 @@ Output and validation types used across commands. All defined in `src/output.rs` pub enum OutputFormat { Text, Json } // output.rs:7 ``` -Global `--output`/`-o` flag. Default `Text`. JSON is free via `#[derive(Serialize)]` on all outcome structs. +Global `--output`/`-o` flag. Default `Text`. JSON is free via +`#[derive(Serialize)]` on all outcome structs. ## Field Hints @@ -20,7 +22,8 @@ pub enum FieldHint { // output.rs:16 } ``` -`field_hints(name)` at `output.rs:39` detects special characters in field names and suggests escaping for `--where` queries. Used in `info` and `check` output. +`field_hints(name)` at `output.rs:39` detects special characters in field names +and suggests escaping for `--where` queries. Used in `info` and `check` output. ## Discovered Field @@ -97,7 +100,11 @@ pub struct FieldViolation { // output.rs:197 Used in `CheckOutcome.violations` and `ValidateOutcome.violations`. -`PartialOrd, Ord` on `ViolationKind` are deliberate: `collect_violations` sorts the output `Vec` by `(field, kind, rule)` (with `files` inner-sorted by path) so `mdvs check` output is byte-stable across runs. The declaration order above is the sort order — adding a variant changes that order; check downstream consumers (diff tooling, golden fixtures) before reordering. +`PartialOrd, Ord` on `ViolationKind` are deliberate: `collect_violations` sorts +the output `Vec` by `(field, kind, rule)` (with `files` +inner-sorted by path) so `mdvs check` output is byte-stable across runs. The +declaration order above is the sort order — adding a variant changes that order; +check downstream consumers (diff tooling, golden fixtures) before reordering. ## New Field @@ -108,25 +115,30 @@ pub struct NewField { // output.rs:210 } ``` -Informational — fields in frontmatter but not in `mdvs.toml`. Does not affect exit code. +Informational — fields in frontmatter but not in `mdvs.toml`. Does not affect +exit code. ## Constraint Violations (post-Wave-B) -Constraint violations are no longer carried as a separate internal type. The `jsonschema` crate emits `ValidationError` instances at validation time; `cmd/check.rs::map_validation_error` translates each into a `ViolationKind` + rule string + per-file detail. +Constraint violations are no longer carried as a separate internal type. The +`jsonschema` crate emits `ValidationError` instances at validation time; +`cmd/check.rs::map_validation_error` translates each into a `ViolationKind` + +rule string + per-file detail. Mapping summary (exhaustive in code): -| jsonschema error | mdvs `ViolationKind` | -|---|---| -| `Type` (non-null mismatch) / `Pattern` | `WrongType` | -| `Type` (null on non-nullable) | `NullNotAllowed` | -| `Enum`, `Constant` | `InvalidCategory` | -| `Minimum`, `Maximum`, `ExclusiveMinimum`, `ExclusiveMaximum`, `MultipleOf` | `OutOfRange` | -| `MinLength`, `MaxLength`, `MinItems`, `MaxItems`, `UniqueItems` | `OutOfRange` | -| `Required` | `MissingRequired` | -| `AdditionalProperties` | `Disallowed` | - -See [architecture.md](./architecture.md#validation-pipeline) for the full pipeline. +| jsonschema error | mdvs `ViolationKind` | +| -------------------------------------------------------------------------- | -------------------- | +| `Type` (non-null mismatch) / `Pattern` | `WrongType` | +| `Type` (null on non-nullable) | `NullNotAllowed` | +| `Enum`, `Constant` | `InvalidCategory` | +| `Minimum`, `Maximum`, `ExclusiveMinimum`, `ExclusiveMaximum`, `MultipleOf` | `OutOfRange` | +| `MinLength`, `MaxLength`, `MinItems`, `MaxItems`, `UniqueItems` | `OutOfRange` | +| `Required` | `MissingRequired` | +| `AdditionalProperties` | `Disallowed` | + +See [architecture.md](./architecture.md#validation-pipeline) for the full +pipeline. ## Build File Detail diff --git a/docs/spec/storage.md b/docs/spec/storage.md index a235afe..ae69760 100644 --- a/docs/spec/storage.md +++ b/docs/spec/storage.md @@ -1,8 +1,14 @@ # Storage -Deep-dive into the Lance storage layer. For the module map see [architecture.md](./architecture.md). +Deep-dive into the Lance storage layer. For the module map see +[architecture.md](./architecture.md). -The storage layer bridges validation (TOML config) and search (LanceDB index). Key files: `index/storage.rs` (Arrow batch construction, column constants, `BuildMetadata`, `content_hash`), `index/backend/` (`LanceBackend`: connection, `write_index` for full rebuilds, `write_index_incremental` for the delete+append delta path, `search`, `--where` translator; the directory is split into `mod.rs` + `read.rs` + `search.rs`). +The storage layer bridges validation (TOML config) and search (LanceDB index). +Key files: `index/storage.rs` (Arrow batch construction, column constants, +`BuildMetadata`, `content_hash`), `index/backend/` (`LanceBackend`: connection, +`write_index` for full rebuilds, `write_index_incremental` for the delete+append +delta path, `search`, `--where` translator; the directory is split into +`mod.rs` + `read.rs` + `search.rs`). ## One Artifact @@ -11,62 +17,97 @@ The storage layer bridges validation (TOML config) and search (LanceDB index). K index.lance/ — Lance dataset, one row per chunk, plus FTS + (optional) vector indexes ``` -A single Lance dataset (table name `index`) holds everything. Each row corresponds to one chunk; per-file fields (`filepath`, `data`, `content_hash`, `built_at`) are duplicated onto each of that file's chunk rows. There is no separate manifest file: per-build configuration lives as table-level key-value metadata on the dataset. +A single Lance dataset (table name `index`) holds everything. Each row +corresponds to one chunk; per-file fields (`filepath`, `data`, `content_hash`, +`built_at`) are duplicated onto each of that file's chunk rows. There is no +separate manifest file: per-build configuration lives as table-level key-value +metadata on the dataset. -This collapsed layout replaces the earlier two-file design (`files.parquet` + `chunks.parquet`) — Lance's single-table-with-indexes model is the more idiomatic fit, and persisting `chunk_text` on the row makes both BM25 full-text indexing and verbose-mode snippet display trivial. +This collapsed layout replaces the earlier two-file design (`files.parquet` + +`chunks.parquet`) — Lance's single-table-with-indexes model is the more +idiomatic fit, and persisting `chunk_text` on the row makes both BM25 full-text +indexing and verbose-mode snippet display trivial. ## Column Layout Column constants live at the top of `index/storage.rs`: -| Column | Constant | Arrow Type | Notes | -|---|---|---|---| -| `chunk_id` | `COL_CHUNK_ID` | Utf8 | UUID | -| `file_id` | `COL_FILE_ID` | Utf8 | UUID, stable across incremental builds for unchanged files | -| `chunk_index` | `COL_CHUNK_INDEX` | Int32 | Zero-based within file | -| `start_line` | `COL_START_LINE` | Int32 | 1-based in source | -| `end_line` | `COL_END_LINE` | Int32 | 1-based, inclusive | -| `chunk_text` | `COL_CHUNK_TEXT` | Utf8 | The plain-text chunk body. Persisted so the BM25 index can tokenize it and verbose snippets can read it without a second file open. | -| `embedding` | `COL_EMBEDDING` | FixedSizeList | Dimension from the active embedding model | -| `filepath` | `COL_FILEPATH` | Utf8 | Relative path from project root (duplicated per chunk) | -| `content_hash` | `COL_CONTENT_HASH` | Utf8 | xxh3-64 hex of the markdown body (duplicated per chunk) | -| `data` | `COL_DATA` | Struct | Frontmatter as a nested Struct (see below) | -| `built_at` | `COL_BUILT_AT` | Timestamp(Microsecond, UTC) | Build time | - -On-disk column names are always the literal constants above. `[search].internal_prefix` and `[search.aliases]` only affect how bare names in `--where` clauses are resolved by the translator (see [search.md](./search.md#collision-avoidance)), not what gets written to disk. +| Column | Constant | Arrow Type | Notes | +| -------------- | ------------------ | --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `chunk_id` | `COL_CHUNK_ID` | Utf8 | UUID | +| `file_id` | `COL_FILE_ID` | Utf8 | UUID, stable across incremental builds for unchanged files | +| `chunk_index` | `COL_CHUNK_INDEX` | Int32 | Zero-based within file | +| `start_line` | `COL_START_LINE` | Int32 | 1-based in source | +| `end_line` | `COL_END_LINE` | Int32 | 1-based, inclusive | +| `chunk_text` | `COL_CHUNK_TEXT` | Utf8 | The plain-text chunk body. Persisted so the BM25 index can tokenize it and verbose snippets can read it without a second file open. | +| `embedding` | `COL_EMBEDDING` | FixedSizeList | Dimension from the active embedding model | +| `filepath` | `COL_FILEPATH` | Utf8 | Relative path from project root (duplicated per chunk) | +| `content_hash` | `COL_CONTENT_HASH` | Utf8 | xxh3-64 hex of the markdown body (duplicated per chunk) | +| `data` | `COL_DATA` | Struct | Frontmatter as a nested Struct (see below) | +| `built_at` | `COL_BUILT_AT` | Timestamp(Microsecond, UTC) | Build time | + +On-disk column names are always the literal constants above. +`[search].internal_prefix` and `[search.aliases]` only affect how bare names in +`--where` clauses are resolved by the translator (see +[search.md](./search.md#collision-avoidance)), not what gets written to disk. ## Indexes Inside the Dataset `LanceBackend::build_indexes()` runs after the table is written: -- **FTS (BM25)** on `chunk_text` — always built. Powers `--mode fulltext` and the hybrid mode's lexical leg. -- **Cosine IVF-PQ** on `embedding` — built only when `n_chunks >= VECTOR_INDEX_MIN_ROWS = 10_000`. Smaller vaults rely on LanceDB's exact flat scan via `nearest_to`, which is plenty fast at that scale. +- **FTS (BM25)** on `chunk_text` — always built. Powers `--mode fulltext` and + the hybrid mode's lexical leg. +- **Cosine IVF-PQ** on `embedding` — built only when + `n_chunks >= VECTOR_INDEX_MIN_ROWS = 10_000`. Smaller vaults rely on LanceDB's + exact flat scan via `nearest_to`, which is plenty fast at that scale. ## The `data` Struct Column -The `data` column is a nested Arrow Struct whose children mirror the source frontmatter's natural shape (YAML mapping, TOML table, or JSON object — all three deserialize to the same JSON shape, which the storage layer transposes into Arrow). A key like `calibration.baseline.wavelength` lands inside a `calibration` Struct child that holds a `baseline` Struct child holding a `wavelength` Float leaf. This lets LanceDB's SQL filter handle `data.calibration.baseline.wavelength > 800` natively via struct field access. - -`build_files_batch()` in `index/storage.rs` produces this shape in two steps (post Wave C / TODO-0097): - -1. **Transpose** the flat list of dotted-name `(name, FieldType)` entries from `mdvs.toml` into a synthetic `FieldType::Object` tree via `transpose_to_storage_type`. This reconstructs the canonical schema's natural shape. -2. **Recurse** via `build_array` against the synthesized tree, passing each file's whole frontmatter Value as the per-row input. The existing Object arm walks `properties.calibration.properties.baseline.properties.wavelength` and assembles the corresponding nested `StructArray` columns. +The `data` column is a nested Arrow Struct whose children mirror the source +frontmatter's natural shape (YAML mapping, TOML table, or JSON object — all +three deserialize to the same JSON shape, which the storage layer transposes +into Arrow). A key like `calibration.baseline.wavelength` lands inside a +`calibration` Struct child that holds a `baseline` Struct child holding a +`wavelength` Float leaf. This lets LanceDB's SQL filter handle +`data.calibration.baseline.wavelength > 800` natively via struct field access. + +`build_files_batch()` in `index/storage.rs` produces this shape in two steps +(post Wave C / TODO-0097): + +1. **Transpose** the flat list of dotted-name `(name, FieldType)` entries from + `mdvs.toml` into a synthetic `FieldType::Object` tree via + `transpose_to_storage_type`. This reconstructs the canonical schema's natural + shape. +2. **Recurse** via `build_array` against the synthesized tree, passing each + file's whole frontmatter Value as the per-row input. The existing Object arm + walks `properties.calibration.properties.baseline.properties.wavelength` and + assembles the corresponding nested `StructArray` columns. `build_array()` handles the FieldType→Arrow mapping recursively: -| FieldType | Arrow Array | Conversion | -|---|---|---| -| Boolean | BooleanArray | `v.as_bool()` | -| Integer | Int64Array | `v.as_i64()` | -| Float | Float64Array | `v.as_f64()`, falls back to `v.as_i64() as f64` | -| String | StringArray | actual strings preserved; non-strings serialized to JSON repr | -| Date | Date32Array | `chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d")`, encoded as days since 1970-01-01. Unparseable values → NULL (defensive; jsonschema's `format: date` already rejects them upstream) | -| DateTime | TimestampMillisecondArray (tz = "UTC") | `chrono::DateTime::parse_from_rfc3339(s)` → `with_timezone(&Utc).timestamp_millis()`. Offsets normalized to UTC; the original offset is intentionally not preserved. Unparseable values → NULL | -| Array(inner) | ListArray | variable-length, child built recursively via `build_array` | -| Object(fields) | StructArray | nested Struct, children built recursively. Reached only via the synthesized storage tree's intermediates (Wave C transposes flat dotted-name leaves back into a nested Object before Arrow encoding). `Array(Object{...})` is rejected on the disk surface (TODO-0155), so no on-disk type produces this arm directly. | - -**Per-row validity** follows the data: a file with `calibration: null` (or no `calibration` key) sees the `calibration` Struct column's validity bit set to 0 for that row, propagating to all descendant columns. A file with `calibration: {baseline: {intensity: 0.5}}` but no `wavelength` leaf sees the leaf's validity bit set to 0 while the intermediate Structs are valid. - -**String preprocessing**: a `String` field is strict by default — non-string JSON values violate validation and never reach the storage layer. Fields declaring `preprocess = ["coerce-to-string"]` (often auto-inferred when mixed types were observed) accept any JSON value; non-strings are serialized to their JSON string representation before validation, then stored as strings. This preserves the "never silently drop data" contract for fields that opt in. +| FieldType | Arrow Array | Conversion | +| -------------- | -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Boolean | BooleanArray | `v.as_bool()` | +| Integer | Int64Array | `v.as_i64()` | +| Float | Float64Array | `v.as_f64()`, falls back to `v.as_i64() as f64` | +| String | StringArray | actual strings preserved; non-strings serialized to JSON repr | +| Date | Date32Array | `chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d")`, encoded as days since 1970-01-01. Unparseable values → NULL (defensive; jsonschema's `format: date` already rejects them upstream) | +| DateTime | TimestampMillisecondArray (tz = "UTC") | `chrono::DateTime::parse_from_rfc3339(s)` → `with_timezone(&Utc).timestamp_millis()`. Offsets normalized to UTC; the original offset is intentionally not preserved. Unparseable values → NULL | +| Array(inner) | ListArray | variable-length, child built recursively via `build_array` | +| Object(fields) | StructArray | nested Struct, children built recursively. Reached only via the synthesized storage tree's intermediates (Wave C transposes flat dotted-name leaves back into a nested Object before Arrow encoding). `Array(Object{...})` is rejected on the disk surface (TODO-0155), so no on-disk type produces this arm directly. | + +**Per-row validity** follows the data: a file with `calibration: null` (or no +`calibration` key) sees the `calibration` Struct column's validity bit set to 0 +for that row, propagating to all descendant columns. A file with +`calibration: {baseline: {intensity: 0.5}}` but no `wavelength` leaf sees the +leaf's validity bit set to 0 while the intermediate Structs are valid. + +**String preprocessing**: a `String` field is strict by default — non-string +JSON values violate validation and never reach the storage layer. Fields +declaring `preprocess = ["coerce-to-string"]` (often auto-inferred when mixed +types were observed) accept any JSON value; non-strings are serialized to their +JSON string representation before validation, then stored as strings. This +preserves the "never silently drop data" contract for fields that opt in. ## Content Hash @@ -82,33 +123,52 @@ pub fn content_hash(content: &str) -> String { - Algorithm: xxHash3-64 - Output: 16-character hex string -Frontmatter-only changes (editing a `status` field) do NOT trigger re-embedding. The hash covers the body that gets chunked and embedded. The `data` column is still rewritten on every chunk row at build time so frontmatter edits are reflected even when no embedding work happens. +Frontmatter-only changes (editing a `status` field) do NOT trigger re-embedding. +The hash covers the body that gets chunked and embedded. The `data` column is +still rewritten on every chunk row at build time so frontmatter edits are +reflected even when no embedding work happens. ## Build Metadata `BuildMetadata` in `index/storage.rs` stores the build configuration snapshot: -| Key | Source | -|---|---| -| `mdvs.provider` | `EmbeddingModelConfig.provider` | -| `mdvs.model` | `EmbeddingModelConfig.name` | -| `mdvs.revision` | `EmbeddingModelConfig.revision` | -| `mdvs.chunk_size` | `ChunkingConfig.max_chunk_size` | -| `mdvs.glob` | `ScanConfig.glob` | -| `mdvs.built_at` | ISO 8601 timestamp | +| Key | Source | +| ------------------ | ---------------------------------------------------------------------- | +| `mdvs.provider` | `EmbeddingModelConfig.provider` | +| `mdvs.model` | `EmbeddingModelConfig.name` | +| `mdvs.revision` | `EmbeddingModelConfig.revision` | +| `mdvs.chunk_size` | `ChunkingConfig.max_chunk_size` | +| `mdvs.glob` | `ScanConfig.glob` | +| `mdvs.built_at` | ISO 8601 timestamp | | `mdvs.schema_hash` | xxh3-64 hex of `dsl_to_canonical(config)` serialized as canonical JSON | -Stored as table-level key-value metadata on the Lance dataset. The full-rebuild path (`LanceBackend::write_index()`) routes the keys through the Arrow `Schema::metadata` map handed to `create_table`; the incremental path (`LanceBackend::write_index_incremental()`) refreshes them on the existing table via `NativeTable::replace_schema_metadata`. Both are read back via `LanceBackend::read_metadata()`. - -**Schema hash** detects field-level changes (types, constraints, path-scoping, preprocessors) that don't show up in any of the other keys. Computed via `compute_schema_hash(config)` in `index/storage.rs`. Hashing the post-translation canonical JSON makes it whitespace-insensitive and key-order-insensitive. Pre-Wave-B datasets without this key read as `""` → treated as changed (conservative, requires `--force`). - -**Config change detection**: build compares current config against stored `BuildMetadata` using `PartialEq`. Mismatch → requires `--force` for full rebuild. The schema-hash mismatch error reads: `"schema: fields, types, constraints, path-scoping, or preprocessors have changed"`. Search compares model identity → hard error on mismatch. +Stored as table-level key-value metadata on the Lance dataset. The full-rebuild +path (`LanceBackend::write_index()`) routes the keys through the Arrow +`Schema::metadata` map handed to `create_table`; the incremental path +(`LanceBackend::write_index_incremental()`) refreshes them on the existing table +via `NativeTable::replace_schema_metadata`. Both are read back via +`LanceBackend::read_metadata()`. + +**Schema hash** detects field-level changes (types, constraints, path-scoping, +preprocessors) that don't show up in any of the other keys. Computed via +`compute_schema_hash(config)` in `index/storage.rs`. Hashing the +post-translation canonical JSON makes it whitespace-insensitive and +key-order-insensitive. Pre-Wave-B datasets without this key read as `""` → +treated as changed (conservative, requires `--force`). + +**Config change detection**: build compares current config against stored +`BuildMetadata` using `PartialEq`. Mismatch → requires `--force` for full +rebuild. The schema-hash mismatch error reads: +`"schema: fields, types, constraints, path-scoping, or preprocessors have changed"`. +Search compares model identity → hard error on mismatch. ## Incremental Build ### Classification -`FileIndexEntry` in `index/storage.rs` is a lightweight projected read (only the columns needed to classify; the expensive `data` Struct + `embedding` columns are not fetched): +`FileIndexEntry` in `index/storage.rs` is a lightweight projected read (only the +columns needed to classify; the expensive `data` Struct + `embedding` columns +are not fetched): ```rust pub struct FileIndexEntry { @@ -120,34 +180,69 @@ pub struct FileIndexEntry { Classification in `cmd/build.rs` compares scanned files against the index: -| Classification | Condition | Action | -|---|---|---| -| **New** | filename not in index | Generate new file_id, chunk, embed | -| **Edited** | filename in index, hash differs | Keep file_id, re-chunk, re-embed | -| **Unchanged** | filename in index, hash matches | Skip chunking/embedding, retain existing chunks | -| **Removed** | in index, not in scan | Drop from output | +| Classification | Condition | Action | +| -------------- | ------------------------------- | ----------------------------------------------- | +| **New** | filename not in index | Generate new file_id, chunk, embed | +| **Edited** | filename in index, hash differs | Keep file_id, re-chunk, re-embed | +| **Unchanged** | filename in index, hash matches | Skip chunking/embedding, retain existing chunks | +| **Removed** | in index, not in scan | Drop from output | ### Write Strategy -`cmd/build/write.rs::write_index_step` selects one of three paths based on the classification result: - -**Skip** — when the build is not a full rebuild AND no files were removed AND no new chunks were produced. Returns `WriteOutcome::Skipped`; no Lance dataset write happens. The skip predicate uses chunk count (not file count) because empty-body files (e.g. Hugo `_index.md`) are always classified as needing embedding but produce zero chunks. - -**Full overwrite** — when `full_rebuild` is true (first build or `--force`). `Backend::write_index` builds one Arrow `RecordBatch` from the retained + new chunks combined and calls `Connection::create_table(...).mode(CreateTableMode::Overwrite)`. The FTS index on `chunk_text` and, above 10k chunks, the IVF-PQ vector index on `embedding` are rebuilt inside the new table. - -**Incremental** — when there is a delta to persist but the table already exists. `Backend::write_index_incremental` opens the existing table, `delete("file_id IN (...)")` for `file_ids_to_clear` (= new + edited + removed file_ids), `add(new_chunks_batch)` for the freshly embedded chunks (the slice past `retained_chunks_count`), calls `NativeTable::replace_schema_metadata` to refresh the `BuildMetadata` keys, and runs `optimize(OptimizeAction::All)` so the existing FTS + vector indexes incorporate the delta without a full rebuild. The retained chunks already in the table are left in place — no rewrite. - -Model loading is skipped entirely when `needs_embedding == 0` (all files unchanged). The write itself is skipped under the further condition above. +`cmd/build/write.rs::write_index_step` selects one of three paths based on the +classification result: + +**Skip** — when the build is not a full rebuild AND no files were removed AND no +new chunks were produced. Returns `WriteOutcome::Skipped`; no Lance dataset +write happens. The skip predicate uses chunk count (not file count) because +empty-body files (e.g. Hugo `_index.md`) are always classified as needing +embedding but produce zero chunks. + +**Full overwrite** — when `full_rebuild` is true (first build or `--force`). +`Backend::write_index` builds one Arrow `RecordBatch` from the retained + new +chunks combined and calls +`Connection::create_table(...).mode(CreateTableMode::Overwrite)`. The FTS index +on `chunk_text` and, above 10k chunks, the IVF-PQ vector index on `embedding` +are rebuilt inside the new table. + +**Incremental** — when there is a delta to persist but the table already exists. +`Backend::write_index_incremental` opens the existing table, +`delete("file_id IN (...)")` for `file_ids_to_clear` (= new + edited + removed +file_ids), `add(new_chunks_batch)` for the freshly embedded chunks (the slice +past `retained_chunks_count`), calls `NativeTable::replace_schema_metadata` to +refresh the `BuildMetadata` keys, and runs `optimize(OptimizeAction::All)` so +the existing FTS + vector indexes incorporate the delta without a full rebuild. +The retained chunks already in the table are left in place — no rewrite. + +Model loading is skipped entirely when `needs_embedding == 0` (all files +unchanged). The write itself is skipped under the further condition above. ## Backend Abstraction -`Backend` enum at `index/backend/mod.rs` has a single variant: `Backend::Lance(LanceBackend)`. The enum is kept (rather than collapsing to a struct) for forward compatibility with future remote-backend work and to keep the existing `LanceBackend::method()` call sites stable. The directory is split into `mod.rs` (the enum + the public method shells), `read.rs` (read-only projections), and `search.rs` (mode-dispatched query + `--where` translator). +`Backend` enum at `index/backend/mod.rs` has a single variant: +`Backend::Lance(LanceBackend)`. The enum is kept (rather than collapsing to a +struct) for forward compatibility with future remote-backend work and to keep +the existing `LanceBackend::method()` call sites stable. The directory is split +into `mod.rs` (the enum + the public method shells), `read.rs` (read-only +projections), and `search.rs` (mode-dispatched query + `--where` translator). `LanceBackend` derives paths from root: + - `.mdvs/` — index directory - `.mdvs/index.lance/` — Lance dataset (the table is named `index`) Key methods: -- `write_index()` — full rebuild path: builds the combined Arrow batch and calls `create_table(...).mode(Overwrite)`, then `build_indexes()`. Used on the first build and whenever `--force` is passed. -- `write_index_incremental()` — delta path: opens the existing table, deletes the rows for `file_ids_to_clear`, appends the new-chunks slice, refreshes the schema metadata via `NativeTable::replace_schema_metadata`, and runs `optimize(All)` so FTS + vector indexes pick up the delta without a full rebuild. -- `read_metadata()` (parses `BuildMetadata` from the Lance table-level kv), `read_file_index()` (lightweight projection for classification), `read_chunk_rows()` (full chunk rows for retained-file pass-through), `search()` (mode-dispatched LanceDB query + best-chunk-per-file dedupe), `stats()`, `clean()`. + +- `write_index()` — full rebuild path: builds the combined Arrow batch and calls + `create_table(...).mode(Overwrite)`, then `build_indexes()`. Used on the first + build and whenever `--force` is passed. +- `write_index_incremental()` — delta path: opens the existing table, deletes + the rows for `file_ids_to_clear`, appends the new-chunks slice, refreshes the + schema metadata via `NativeTable::replace_schema_metadata`, and runs + `optimize(All)` so FTS + vector indexes pick up the delta without a full + rebuild. +- `read_metadata()` (parses `BuildMetadata` from the Lance table-level kv), + `read_file_index()` (lightweight projection for classification), + `read_chunk_rows()` (full chunk rows for retained-file pass-through), + `search()` (mode-dispatched LanceDB query + best-chunk-per-file dedupe), + `stats()`, `clean()`. diff --git a/docs/spec/todos/TODO-0001.md b/docs/spec/todos/TODO-0001.md index 1c815a3..cf3ebec 100644 --- a/docs/spec/todos/TODO-0001.md +++ b/docs/spec/todos/TODO-0001.md @@ -16,13 +16,21 @@ blocks: [] ## Summary -Null values should be transparent during type inference — they lose to any real type. Only if every occurrence of a field is null should the type fall back to String. +Null values should be transparent during type inference — they lose to any real +type. Only if every occurrence of a field is null should the type fall back to +String. ## Resolution -Used Approach A: filter nulls in `infer_field_types()` rather than adding a `FieldType::Null` variant. +Used Approach A: filter nulls in `infer_field_types()` rather than adding a +`FieldType::Null` variant. -- `src/discover/infer.rs`: Skip `Value::Null` during type accumulation in `infer_field_types()`. Null-only fields default to String after the loop. Two new tests: `null_transparent_in_widening`, `null_plus_int_infers_int`. -- `src/cmd/check.rs`: Null values treated as absent — skipped during type/allowed validation, treated as missing for required validation. (Temporary measure; TODO-0005 tracks future differentiation.) +- `src/discover/infer.rs`: Skip `Value::Null` during type accumulation in + `infer_field_types()`. Null-only fields default to String after the loop. Two + new tests: `null_transparent_in_widening`, `null_plus_int_infers_int`. +- `src/cmd/check.rs`: Null values treated as absent — skipped during + type/allowed validation, treated as missing for required validation. + (Temporary measure; TODO-0005 tracks future differentiation.) -Validated on a 469-file vault: `projects` correctly inferred as `String[]`, clean check. +Validated on a 469-file vault: `projects` correctly inferred as `String[]`, +clean check. diff --git a/docs/spec/todos/TODO-0002.md b/docs/spec/todos/TODO-0002.md index f505e6b..ced78be 100644 --- a/docs/spec/todos/TODO-0002.md +++ b/docs/spec/todos/TODO-0002.md @@ -25,8 +25,15 @@ files_updated: ## Summary -Replace `walkdir` with the `ignore` crate in `scan.rs` to support `.mdvsignore` files and `.gitignore` integration. Add `skip_gitignore` setting to `[scan]` config. +Replace `walkdir` with the `ignore` crate in `scan.rs` to support `.mdvsignore` +files and `.gitignore` integration. Add `skip_gitignore` setting to `[scan]` +config. ## Resolution -Replaced `walkdir` with `ignore` crate's `WalkBuilder`. `.mdvsignore` files are always read at all directory levels. `.gitignore` is read by default and can be disabled via `skip_gitignore` in `[scan]` config or `--skip-gitignore` CLI flag on `init`. Refactored `ScannedFiles::scan()` to take `&ScanConfig` instead of individual params. Updated `MdvsToml::from_inferred()` to accept `ScanConfig` directly. All callers and tests updated. +Replaced `walkdir` with `ignore` crate's `WalkBuilder`. `.mdvsignore` files are +always read at all directory levels. `.gitignore` is read by default and can be +disabled via `skip_gitignore` in `[scan]` config or `--skip-gitignore` CLI flag +on `init`. Refactored `ScannedFiles::scan()` to take `&ScanConfig` instead of +individual params. Updated `MdvsToml::from_inferred()` to accept `ScanConfig` +directly. All callers and tests updated. diff --git a/docs/spec/todos/TODO-0003.md b/docs/spec/todos/TODO-0003.md index b2c534c..cef1b5b 100644 --- a/docs/spec/todos/TODO-0003.md +++ b/docs/spec/todos/TODO-0003.md @@ -15,8 +15,11 @@ files_updated: ## Summary -Replace `--auto-build` (bool flag, always true) with `--suppress-auto-build` so users can disable auto-build. Default behavior remains auto-build enabled. +Replace `--auto-build` (bool flag, always true) with `--suppress-auto-build` so +users can disable auto-build. Default behavior remains auto-build enabled. ## Resolution -Replaced `--auto-build` (with `default_value = "true"`) with `--suppress-auto-build` boolean flag. Default absent = auto-build enabled. When present, `!suppress_auto_build` is passed to `init::run()`. +Replaced `--auto-build` (with `default_value = "true"`) with +`--suppress-auto-build` boolean flag. Default absent = auto-build enabled. When +present, `!suppress_auto_build` is passed to `init::run()`. diff --git a/docs/spec/todos/TODO-0004.md b/docs/spec/todos/TODO-0004.md index 40c2306..4f15863 100644 --- a/docs/spec/todos/TODO-0004.md +++ b/docs/spec/todos/TODO-0004.md @@ -19,4 +19,6 @@ Rename the search `--where-clause` flag to `--where` to match the spec. ## Resolution -Changed `#[arg(long, name = "where")]` to `#[arg(long = "where")]` in the Search command definition. The `name` attribute only sets the value placeholder, not the flag name — `long = "where"` is what controls the CLI flag. +Changed `#[arg(long, name = "where")]` to `#[arg(long = "where")]` in the Search +command definition. The `name` attribute only sets the value placeholder, not +the flag name — `long = "where"` is what controls the CLI flag. diff --git a/docs/spec/todos/TODO-0005.md b/docs/spec/todos/TODO-0005.md index 9e2d273..4bd11c3 100644 --- a/docs/spec/todos/TODO-0005.md +++ b/docs/spec/todos/TODO-0005.md @@ -12,13 +12,21 @@ blocks: [] ## Summary -Currently null frontmatter values (`field:` with no value) are treated identically to absent fields in check. Null and absent are two distinct states — a field with a null value has the key present, and a field that is absent has no key at all. This distinction matters for required validation: a required field with a null value should pass if the field is nullable. +Currently null frontmatter values (`field:` with no value) are treated +identically to absent fields in check. Null and absent are two distinct states — +a field with a null value has the key present, and a field that is absent has no +key at all. This distinction matters for required validation: a required field +with a null value should pass if the field is nullable. ## Details ### The bug -Inference counts null-valued keys as "present" when computing required/allowed globs (correct — the key exists in the frontmatter). But check treats null as "absent", triggering MissingRequired. This violates the init-must-pass invariant: `init` infers a schema that immediately fails `check` against the same files. +Inference counts null-valued keys as "present" when computing required/allowed +globs (correct — the key exists in the frontmatter). But check treats null as +"absent", triggering MissingRequired. This violates the init-must-pass +invariant: `init` infers a schema that immediately fails `check` against the +same files. ### Design @@ -30,26 +38,30 @@ Add `nullable: bool` to the field schema. Three orthogonal axes per field: Semantics in check: -| State | required + nullable | required + not nullable | -|---|---|---| -| Key absent | MissingRequired | MissingRequired | -| Key present, value null | pass | NullNotAllowed (new violation) | -| Key present, value non-null | pass (type check applies) | pass (type check applies) | +| State | required + nullable | required + not nullable | +| --------------------------- | ------------------------- | ------------------------------ | +| Key absent | MissingRequired | MissingRequired | +| Key present, value null | pass | NullNotAllowed (new violation) | +| Key present, value non-null | pass (type check applies) | pass (type check applies) | ### Inference -- **required/allowed**: null-valued keys count as "present" (already the case — `infer_field_types` adds null files to the `files` list, `DirectoryTree` uses `map.keys()` which includes null-valued keys) -- **type**: null is transparent (already the case — null values are skipped during widening, null-only fields default to String) -- **nullable**: `true` if any file has a null value for the field, `false` otherwise (new) +- **required/allowed**: null-valued keys count as "present" (already the case — + `infer_field_types` adds null files to the `files` list, `DirectoryTree` uses + `map.keys()` which includes null-valued keys) +- **type**: null is transparent (already the case — null values are skipped + during widening, null-only fields default to String) +- **nullable**: `true` if any file has a null value for the field, `false` + otherwise (new) ### Changes -| File | Change | -|---|---| -| `src/discover/infer.rs` | Compute `nullable` in `InferredField` (any null → true) | -| `src/schema/config.rs` | Add `nullable: bool` to `TomlField` (default false) | -| `src/cmd/check.rs` | Treat null as "key present" for required check; add `NullNotAllowed` violation when nullable=false | -| `src/output.rs` | Add `nullable` to `DiscoveredField` | -| `src/cmd/init.rs` | Write `nullable` to toml | -| `src/cmd/update.rs` | Infer `nullable` for new/reinferred fields | -| `src/cmd/info.rs` | Display `nullable` | +| File | Change | +| ----------------------- | -------------------------------------------------------------------------------------------------- | +| `src/discover/infer.rs` | Compute `nullable` in `InferredField` (any null → true) | +| `src/schema/config.rs` | Add `nullable: bool` to `TomlField` (default false) | +| `src/cmd/check.rs` | Treat null as "key present" for required check; add `NullNotAllowed` violation when nullable=false | +| `src/output.rs` | Add `nullable` to `DiscoveredField` | +| `src/cmd/init.rs` | Write `nullable` to toml | +| `src/cmd/update.rs` | Infer `nullable` for new/reinferred fields | +| `src/cmd/info.rs` | Display `nullable` | diff --git a/docs/spec/todos/TODO-0006.md b/docs/spec/todos/TODO-0006.md index b1c85d3..ab4fd19 100644 --- a/docs/spec/todos/TODO-0006.md +++ b/docs/spec/todos/TODO-0006.md @@ -13,17 +13,23 @@ blocks: [8, 10, 145] ## Summary -Add a `categories` constraint to fields that restricts values to a declared set of allowed values. Not a new FieldType — categorical is a validation constraint on existing types. Applies to String, Integer, and Array (element-level) fields. Automatically inferred during init/update with manual override support. +Add a `categories` constraint to fields that restricts values to a declared set +of allowed values. Not a new FieldType — categorical is a validation constraint +on existing types. Applies to String, Integer, and Array (element-level) fields. +Automatically inferred during init/update with manual override support. -This TODO also scaffolds the full constraint infrastructure (two-layer architecture, constraint resolver) that TODO-0008 and TODO-0010 build on. +This TODO also scaffolds the full constraint infrastructure (two-layer +architecture, constraint resolver) that TODO-0008 and TODO-0010 build on. ## Constraint architecture -Two-layer design: a flat `Constraints` struct for TOML serialization, and a `ConstraintKind` enum for structured behavior dispatch. +Two-layer design: a flat `Constraints` struct for TOML serialization, and a +`ConstraintKind` enum for structured behavior dispatch. ### Serde layer — `Constraints` struct -Maps directly to the flat `[fields.field.constraints]` TOML table. Each constraint kind is an `Option` field. This is what gets serialized/deserialized. +Maps directly to the flat `[fields.field.constraints]` TOML table. Each +constraint kind is an `Option` field. This is what gets serialized/deserialized. ```rust #[derive(Serialize, Deserialize)] @@ -37,7 +43,9 @@ struct Constraints { ### Behavior layer — `ConstraintKind` enum -Each variant encapsulates one constraint kind's behavior: which types it applies to and how it validates a value. Follows the same enum-dispatch pattern as `FieldType`, `Backend`, and `Embedder`. +Each variant encapsulates one constraint kind's behavior: which types it applies +to and how it validates a value. Follows the same enum-dispatch pattern as +`FieldType`, `Backend`, and `Embedder`. ```rust enum ConstraintKind { @@ -61,11 +69,15 @@ impl ConstraintKind { } ``` -Each constraint kind owns its own error messages — both for "I don't apply to this type" and "I conflict with that other constraint." Adding a new constraint kind is self-contained: implement `validate_for_type` and `conflicts_with` against existing kinds. +Each constraint kind owns its own error messages — both for "I don't apply to +this type" and "I conflict with that other constraint." Adding a new constraint +kind is self-contained: implement `validate_for_type` and `conflicts_with` +against existing kinds. ### Bridge — `Constraints::active()` -Bridges the two layers. Reads which `Option` fields are `Some` and produces the corresponding `ConstraintKind` variants. +Bridges the two layers. Reads which `Option` fields are `Some` and produces the +corresponding `ConstraintKind` variants. ```rust impl Constraints { @@ -82,11 +94,16 @@ impl Constraints { ### Constraint resolver — `Constraints::validate_config()` -Called during config loading (existing `validate_config()` path). Two-phase validation: -1. **Self-validation** — each active constraint checks it's applicable to the field's type and well-formed -2. **Pairwise compatibility** — each pair of active constraints checks mutual compatibility +Called during config loading (existing `validate_config()` path). Two-phase +validation: -Both phases produce descriptive `ConfigError` values owned by the constraint kinds themselves. +1. **Self-validation** — each active constraint checks it's applicable to the + field's type and well-formed +2. **Pairwise compatibility** — each pair of active constraints checks mutual + compatibility + +Both phases produce descriptive `ConfigError` values owned by the constraint +kinds themselves. ```rust impl Constraints { @@ -111,11 +128,18 @@ impl Constraints { } ``` -Compatibility is managed per-pair, not as a blanket mutual-exclusion rule. This is scalable — adding a new constraint kind means adding its pairwise rules against existing kinds. Rules can be relaxed individually without affecting others (e.g., allow `categories` + `min_length` on Array while still blocking `categories` + `min`/`max` on Integer). +Compatibility is managed per-pair, not as a blanket mutual-exclusion rule. This +is scalable — adding a new constraint kind means adding its pairwise rules +against existing kinds. Rules can be relaxed individually without affecting +others (e.g., allow `categories` + `min_length` on Array while still blocking +`categories` + `min`/`max` on Integer). ## TOML representation -Constraints live in a `[fields.field.constraints]` sub-table, not as top-level field attributes. This keeps constraints visually and semantically grouped as refinements of the type, separate from structural attributes like `name`, `type`, `nullable`, `allowed`, `required`. +Constraints live in a `[fields.field.constraints]` sub-table, not as top-level +field attributes. This keeps constraints visually and semantically grouped as +refinements of the type, separate from structural attributes like `name`, +`type`, `nullable`, `allowed`, `required`. ```toml [[fields.field]] @@ -145,7 +169,9 @@ type = "String" # no constraints section — unconstrained ``` -Presence of `categories` makes the field categorical — no separate boolean flag needed. Absence of the `[fields.field.constraints]` section (or an absent `categories` key) means unconstrained. +Presence of `categories` makes the field categorical — no separate boolean flag +needed. Absence of the `[fields.field.constraints]` section (or an absent +`categories` key) means unconstrained. ### Rust representation on `TomlField` @@ -163,23 +189,31 @@ struct TomlField { - **String** — value must be one of the listed strings - **Integer** — value must be one of the listed integers -- **Array(String)** / **Array(Integer)** — each element must be one of the listed values +- **Array(String)** / **Array(Integer)** — each element must be one of the + listed values - **Boolean** — already two-valued, no need - **Float** — continuous, doesn't apply - **Object** — structural, doesn't apply ## Auto-inference -During `init` and `update reinfer`, detect categorical fields using two conditions (both must hold): +During `init` and `update reinfer`, detect categorical fields using two +conditions (both must hold): 1. **Max distinct values** — distinct value count ≤ 10. -2. **Minimum repetition** — `occurrences / distinct_values ≥ 3`. Each value appears at least three times on average. Filters out low-reuse fields. +2. **Minimum repetition** — `occurrences / distinct_values ≥ 3`. Each value + appears at least three times on average. Filters out low-reuse fields. Examples: -- `status` with 3 values across 30 files: distinct=3 (≤10), avg repetition=10 (≥2) → **categorical** -- `priority` with 3 values across 9 files: distinct=3 (≤10), avg repetition=3 (≥3) → **categorical** -- `title` with 28 values across 30 files: distinct=28 (>10) → **not categorical** -- `author` with 5 values across 18 files: distinct=5 (≤10), avg repetition=3 (≥3) → **categorical** + +- `status` with 3 values across 30 files: distinct=3 (≤10), avg repetition=10 + (≥2) → **categorical** +- `priority` with 3 values across 9 files: distinct=3 (≤10), avg repetition=3 + (≥3) → **categorical** +- `title` with 28 values across 30 files: distinct=28 (>10) → **not + categorical** +- `author` with 5 values across 18 files: distinct=5 (≤10), avg repetition=3 + (≥3) → **categorical** ### Configurable thresholds @@ -191,7 +225,8 @@ max_categories = 10 # default: 10 min_category_repetition = 3 # default: 3 ``` -These control automatic inference only. Manual `categories` in toml are unaffected by thresholds. +These control automatic inference only. Manual `categories` in toml are +unaffected by thresholds. CLI flags on `update reinfer` override the toml values per-invocation: @@ -201,7 +236,8 @@ mdvs update reinfer [fields..] --max-categories 15 --min-repetition 3 ## CLI: `update reinfer` subcommand -`reinfer` becomes a subcommand of `update` (not a flag). This cleanly separates the two workflows: +`reinfer` becomes a subcommand of `update` (not a flag). This cleanly separates +the two workflows: ``` mdvs update [path] # default: detect new fields (additive) @@ -211,11 +247,14 @@ mdvs update reinfer [path] [fields..] # re-infer specific fields (or all if --min-repetition # override threshold (default: from toml or 2) ``` -- `update reinfer` with no fields listed = re-infer all (replaces `--reinfer-all`) -- `update reinfer status priority` = re-infer only those fields (replaces `--reinfer `) +- `update reinfer` with no fields listed = re-infer all (replaces + `--reinfer-all`) +- `update reinfer status priority` = re-infer only those fields (replaces + `--reinfer `) - `--categorical` / `--no-categorical` only make sense with named fields -Note: this is a breaking change to the CLI surface. The old `--reinfer`/`--reinfer-all` flags are removed. +Note: this is a breaking change to the CLI surface. The old +`--reinfer`/`--reinfer-all` flags are removed. ## Validation (check) @@ -224,59 +263,86 @@ New violation kind: `InvalidCategory` - **rule**: `categories = ["draft", "published", "archived"]` - **detail**: `got "pending"` -For arrays, validation checks each element. Detail shows the offending element(s). +For arrays, validation checks each element. Detail shows the offending +element(s). -Null values on categorical fields: follow existing nullable logic. If `nullable = true`, null skips the category check. If `nullable = false`, null triggers `NullNotAllowed` (existing behavior, before category check). +Null values on categorical fields: follow existing nullable logic. If +`nullable = true`, null skips the category check. If `nullable = false`, null +triggers `NullNotAllowed` (existing behavior, before category check). -Validation dispatches through `ConstraintKind::validate_value()` — the check command calls `constraints.active()` and validates each constraint kind against the field value. +Validation dispatches through `ConstraintKind::validate_value()` — the check +command calls `constraints.active()` and validates each constraint kind against +the field value. ## Storage -No change. Categorical fields remain their underlying Arrow type (Utf8, Int64, etc.) in Parquet. The constraint is validation-only. +No change. Categorical fields remain their underlying Arrow type (Utf8, Int64, +etc.) in Parquet. The constraint is validation-only. ## Query -No change. `--where` clauses on categorical fields work the same as any String/Integer field. +No change. `--where` clauses on categorical fields work the same as any +String/Integer field. ## Implementation waves ### Wave 1: Constraint types and resolver -Scaffold the two-layer architecture with unit tests. No integration with the rest of the codebase yet. +Scaffold the two-layer architecture with unit tests. No integration with the +rest of the codebase yet. -- `src/schema/constraints.rs` (NEW) — `Constraints` struct, `ConstraintKind` enum, `active()` bridge, `validate_for_type()`, `conflicts_with()`, `validate_config()` resolver, `validate_value()` dispatch -- Tests: serde roundtrip (toml → Constraints → toml), `active()` produces correct variants, `validate_for_type()` accepts/rejects correct field types, `validate_value()` catches invalid category values, `validate_config()` aggregates errors +- `src/schema/constraints.rs` (NEW) — `Constraints` struct, `ConstraintKind` + enum, `active()` bridge, `validate_for_type()`, `conflicts_with()`, + `validate_config()` resolver, `validate_value()` dispatch +- Tests: serde roundtrip (toml → Constraints → toml), `active()` produces + correct variants, `validate_for_type()` accepts/rejects correct field types, + `validate_value()` catches invalid category values, `validate_config()` + aggregates errors ### Wave 2: Config integration Wire `Constraints` into `TomlField` and the existing config validation path. -- `src/schema/config.rs` — add `constraints: Option` to `TomlField`, add `max_categories`/`min_category_repetition` to `FieldsConfig`, call `Constraints::validate_config()` during config loading +- `src/schema/config.rs` — add `constraints: Option` to + `TomlField`, add `max_categories`/`min_category_repetition` to `FieldsConfig`, + call `Constraints::validate_config()` during config loading - `src/schema/mod.rs` — export `constraints` module -- Tests: toml files with constraints parse correctly, invalid constraints (wrong type, conflicting kinds) produce config errors +- Tests: toml files with constraints parse correctly, invalid constraints (wrong + type, conflicting kinds) produce config errors ### Wave 3: Inference Collect distinct values during init/update and apply the categorical heuristic. -- `src/discover/infer.rs` — collect distinct values per field, apply heuristic (distinct ≤ max_categories AND occurrences/distinct ≥ min_repetition), populate `categories` on inferred `TomlField` -- `src/cmd/init.rs` — categorical inference during init (uses thresholds from defaults or CLI) -- Tests: heuristic correctly classifies categorical vs non-categorical fields, thresholds are respected, Array element-level inference works +- `src/discover/infer.rs` — collect distinct values per field, apply heuristic + (distinct ≤ max_categories AND occurrences/distinct ≥ min_repetition), + populate `categories` on inferred `TomlField` +- `src/cmd/init.rs` — categorical inference during init (uses thresholds from + defaults or CLI) +- Tests: heuristic correctly classifies categorical vs non-categorical fields, + thresholds are respected, Array element-level inference works ### Wave 4: Validation -Add `InvalidCategory` violation to check and wire constraint validation into the check pipeline. +Add `InvalidCategory` violation to check and wire constraint validation into the +check pipeline. -- `src/cmd/check.rs` — validate values via `ConstraintKind::validate_value()`, new `InvalidCategory` violation kind -- Tests: check detects invalid categories on String/Integer/Array fields, null handling respects nullable flag, valid categories pass +- `src/cmd/check.rs` — validate values via `ConstraintKind::validate_value()`, + new `InvalidCategory` violation kind +- Tests: check detects invalid categories on String/Integer/Array fields, null + handling respects nullable flag, valid categories pass ### Wave 5: CLI — `update reinfer` subcommand -Refactor `update` to use a subcommand for reinfer, with categorical override flags. +Refactor `update` to use a subcommand for reinfer, with categorical override +flags. -- `src/cmd/update.rs` — `reinfer` subcommand replacing `--reinfer`/`--reinfer-all` flags, add `--categorical`/`--no-categorical`/`--max-categories`/`--min-repetition` flags +- `src/cmd/update.rs` — `reinfer` subcommand replacing + `--reinfer`/`--reinfer-all` flags, add + `--categorical`/`--no-categorical`/`--max-categories`/`--min-repetition` flags - `src/main.rs` — wire up subcommand -- Tests: reinfer applies heuristic, `--categorical` forces categories, `--no-categorical` strips them, threshold flags override toml values +- Tests: reinfer applies heuristic, `--categorical` forces categories, + `--no-categorical` strips them, threshold flags override toml values ### Wave 6: Integration tests @@ -284,13 +350,21 @@ End-to-end tests against `example_kb` and verify the full pipeline. - Add categorical fields to `example_kb` fixtures if needed - Test init → check → update reinfer round-trip -- Verify build still works with categorical fields (no storage changes, but constraints must not interfere) +- Verify build still works with categorical fields (no storage changes, but + constraints must not interfere) ## Files -- `src/schema/constraints.rs` (NEW) — `Constraints` struct, `ConstraintKind` enum, `active()` bridge, `validate_config()` resolver, `validate_value()` dispatch -- `src/schema/config.rs` — add `constraints: Option` to `TomlField`, call `validate_config()` during config validation, add `max_categories`/`min_category_repetition` to `FieldsConfig` -- `src/discover/infer.rs` — collect distinct values, apply heuristic, populate categories -- `src/cmd/check.rs` — validate values via `ConstraintKind::validate_value()`, new `InvalidCategory` violation -- `src/cmd/update.rs` — `reinfer` subcommand with `--categorical`/`--no-categorical`/`--max-categories`/`--min-repetition` flags +- `src/schema/constraints.rs` (NEW) — `Constraints` struct, `ConstraintKind` + enum, `active()` bridge, `validate_config()` resolver, `validate_value()` + dispatch +- `src/schema/config.rs` — add `constraints: Option` to + `TomlField`, call `validate_config()` during config validation, add + `max_categories`/`min_category_repetition` to `FieldsConfig` +- `src/discover/infer.rs` — collect distinct values, apply heuristic, populate + categories +- `src/cmd/check.rs` — validate values via `ConstraintKind::validate_value()`, + new `InvalidCategory` violation +- `src/cmd/update.rs` — `reinfer` subcommand with + `--categorical`/`--no-categorical`/`--max-categories`/`--min-repetition` flags - `src/cmd/init.rs` — categorical inference during init diff --git a/docs/spec/todos/TODO-0007.md b/docs/spec/todos/TODO-0007.md index e9fe165..fbe37c1 100644 --- a/docs/spec/todos/TODO-0007.md +++ b/docs/spec/todos/TODO-0007.md @@ -13,51 +13,88 @@ blocks: [] # TODO-0007: Support Date and DateTime field types > **Closeout (2026-05-14).** Four waves shipped on branch `feat/todo-0007`: -> - **Wave 1** (`26529b1` + `48c25c7` + `90eafec`) — `FieldType::Date`, parser, JSON Schema translator + gate (`format: date`), storage as Arrow `Date32`, jsonschema format validation enabled, constraint applicability, 29 tests. Bundled a separate cleanup of all production-code `unwrap`/`expect`/`panic` calls (41 sites). -> - **Wave 2** (`ede5d4b` + `9a6227b`) — auto-inference: `From<&Value> for FieldType` checks RFC 3339 full-date shape + chrono validation; strict semantics via existing widening. example_kb's `joined`, `date`, `commission_date`, `last_reviewed` auto-promoted. -> - **Wave 3** (`5b6c84e`) — `FieldType::DateTime`, parser, JSON Schema (`format: date-time`), Arrow `Timestamp(Millisecond, "UTC")` with UTC normalization, inference, constraint applicability, 31 tests. example_kb's `synced_at` (added to two experiment files with `Z` and `+02:00` offsets) demonstrates both forms. +> +> - **Wave 1** (`26529b1` + `48c25c7` + `90eafec`) — `FieldType::Date`, parser, +> JSON Schema translator + gate (`format: date`), storage as Arrow `Date32`, +> jsonschema format validation enabled, constraint applicability, 29 tests. +> Bundled a separate cleanup of all production-code `unwrap`/`expect`/`panic` +> calls (41 sites). +> - **Wave 2** (`ede5d4b` + `9a6227b`) — auto-inference: +> `From<&Value> for FieldType` checks RFC 3339 full-date shape + chrono +> validation; strict semantics via existing widening. example_kb's `joined`, +> `date`, `commission_date`, `last_reviewed` auto-promoted. +> - **Wave 3** (`5b6c84e`) — `FieldType::DateTime`, parser, JSON Schema +> (`format: date-time`), Arrow `Timestamp(Millisecond, "UTC")` with UTC +> normalization, inference, constraint applicability, 31 tests. example_kb's +> `synced_at` (added to two experiment files with `Z` and `+02:00` offsets) +> demonstrates both forms. > - **Wave 4** (this commit) — docs + mdBook sweep. Closes the TODO. > -> 775 tests pass. Both types pass the launch-readiness gate; the Date type item in `launch_readiness.md` flips to done. +> 775 tests pass. Both types pass the launch-readiness gate; the Date type item +> in `launch_readiness.md` flips to done. ## Summary -Two new `FieldType` variants — `Date` (calendar date, no time) and `DateTime` (date + time + optional timezone). Today both shapes are inferred as `String`, which is correct but useless: every Obsidian vault has `date:`, `created:`, `published:`, `joined:` fields that deserve typed handling, native date arithmetic in `--where`, and richer validation. +Two new `FieldType` variants — `Date` (calendar date, no time) and `DateTime` +(date + time + optional timezone). Today both shapes are inferred as `String`, +which is correct but useless: every Obsidian vault has `date:`, `created:`, +`published:`, `joined:` fields that deserve typed handling, native date +arithmetic in `--where`, and richer validation. -This TODO is gating the public launch (along with TODO-0143 and the lance backend swap) — see the launch-readiness memory. +This TODO is gating the public launch (along with TODO-0143 and the lance +backend swap) — see the launch-readiness memory. ## Format choices -Standardize on **RFC 3339** as the canonical wire format — strict subset of ISO 8601 designed for machine interoperability. ISO 8601 itself is permissive (accepts `20240115T143000`, week dates `2024-W03-1`, missing-seconds shorthand, etc.) which would force us into ambiguous parsing. RFC 3339 has one canonical shape per type, is what JSON Schema's `format: date` / `format: date-time` validators expect, and matches `chrono::DateTime::parse_from_rfc3339` and `chrono::NaiveDate::parse_from_str` semantics. Non-RFC-3339 formats fall back to `String` with a `pattern` constraint. +Standardize on **RFC 3339** as the canonical wire format — strict subset of ISO +8601 designed for machine interoperability. ISO 8601 itself is permissive +(accepts `20240115T143000`, week dates `2024-W03-1`, missing-seconds shorthand, +etc.) which would force us into ambiguous parsing. RFC 3339 has one canonical +shape per type, is what JSON Schema's `format: date` / `format: date-time` +validators expect, and matches `chrono::DateTime::parse_from_rfc3339` and +`chrono::NaiveDate::parse_from_str` semantics. Non-RFC-3339 formats fall back to +`String` with a `pattern` constraint. -| FieldType | Wire format | JSON Schema `format` | Arrow type | -|---|---|---|---| -| `Date` | `YYYY-MM-DD` | `date` | `Date32` (days since epoch) | -| `DateTime` | `YYYY-MM-DDTHH:MM:SS[.frac][Z\|±offset]` | `date-time` | `Timestamp(Millisecond, Some("UTC"))` (TBC) | +| FieldType | Wire format | JSON Schema `format` | Arrow type | +| ---------- | ---------------------------------------- | -------------------- | ------------------------------------------- | +| `Date` | `YYYY-MM-DD` | `date` | `Date32` (days since epoch) | +| `DateTime` | `YYYY-MM-DDTHH:MM:SS[.frac][Z\|±offset]` | `date-time` | `Timestamp(Millisecond, Some("UTC"))` (TBC) | -Standalone `Time` (`HH:MM:SS`) is **out of scope** — rare in markdown frontmatter; add later if asked. +Standalone `Time` (`HH:MM:SS`) is **out of scope** — rare in markdown +frontmatter; add later if asked. ## Type integration -**On-disk syntax:** `type = "Date"` and `type = "DateTime"`. Extends the parser's `Scalar` alternative; grammar becomes `Scalar | Array(Scalar)` where `Scalar ∈ {String, Integer, Float, Boolean, Date, DateTime}`. +**On-disk syntax:** `type = "Date"` and `type = "DateTime"`. Extends the +parser's `Scalar` alternative; grammar becomes `Scalar | Array(Scalar)` where +`Scalar ∈ {String, Integer, Float, Boolean, Date, DateTime}`. -**Display:** `Date` and `DateTime`. `Array(Date)` and `Array(DateTime)` round-trip naturally. +**Display:** `Date` and `DateTime`. `Array(Date)` and `Array(DateTime)` +round-trip naturally. **Widening:** + - `Date + Date → Date`, `DateTime + DateTime → DateTime`. -- `Date + DateTime → String` (lossy — DateTime has info that doesn't fit Date, and Date isn't a valid DateTime). Strict path, no automatic promotion. +- `Date + DateTime → String` (lossy — DateTime has info that doesn't fit Date, + and Date isn't a valid DateTime). Strict path, no automatic promotion. - `Date|DateTime + String → String`. Once a String is observed, stay String. - `Date|DateTime + non-string → String`. -**Widening matrix update:** new Date and DateTime rows/columns in `book/src/concepts/types.md`. +**Widening matrix update:** new Date and DateTime rows/columns in +`book/src/concepts/types.md`. ## Validation -JSON Schema's `format: date` and `format: date-time` validators do the work. The `jsonschema` crate (v0.46) has format validation as opt-in for draft 2020-12 — we enable it on the Validator builder for these two formats. +JSON Schema's `format: date` and `format: date-time` validators do the work. The +`jsonschema` crate (v0.46) has format validation as opt-in for draft 2020-12 — +we enable it on the Validator builder for these two formats. **Constraints supported in v1:** + - `categories` — explicit allowlist of dates/datetimes. Trivially valid. -- (deferred) `min` / `max` — JSON Schema's `minimum`/`maximum` is for numbers; date bounds need `formatMinimum`/`formatMaximum` (separate vocab) or a Rust-side precheck. Skip in v1, follow-up TODO. +- (deferred) `min` / `max` — JSON Schema's `minimum`/`maximum` is for numbers; + date bounds need `formatMinimum`/`formatMaximum` (separate vocab) or a + Rust-side precheck. Skip in v1, follow-up TODO. - (n/a) `pattern` — the format IS the pattern. - (n/a) `min_length` / `max_length` — not meaningful for date types. @@ -66,92 +103,136 @@ JSON Schema's `format: date` and `format: date-time` validators do the work. The Pattern detection runs on each scanned string value. The decision tree: 1. Scan all non-null observations for a field. -2. If 100% match the `YYYY-MM-DD` regex (with valid month/day ranges): candidate `Date`. +2. If 100% match the `YYYY-MM-DD` regex (with valid month/day ranges): candidate + `Date`. 3. If 100% match the RFC 3339 datetime regex: candidate `DateTime`. 4. If mixed (some Date, some DateTime, or some non-date strings): `String`. 5. Single non-matching observation downgrades to `String` — strict. -Tracked through `FieldTypeInfo.observed_types` (same machinery used by the preprocessor-inference pipeline). +Tracked through `FieldTypeInfo.observed_types` (same machinery used by the +preprocessor-inference pipeline). ## Storage -**Date → Arrow `Date32`** (4 bytes, days since 1970-01-01). Conversion in `storage.rs::build_array`: parse the JSON string via `chrono::NaiveDate::parse_from_str` or Arrow's built-in date string parser; emit `Date32Array`. +**Date → Arrow `Date32`** (4 bytes, days since 1970-01-01). Conversion in +`storage.rs::build_array`: parse the JSON string via +`chrono::NaiveDate::parse_from_str` or Arrow's built-in date string parser; emit +`Date32Array`. -**DateTime → Arrow `Timestamp(Millisecond, Some("UTC"))`** (TBC: unit precision and timezone strategy). Normalize timezone-offset values to UTC at storage time so all rows compare consistently. Timezone-naive datetimes interpreted as UTC (document this). +**DateTime → Arrow `Timestamp(Millisecond, Some("UTC"))`** (TBC: unit precision +and timezone strategy). Normalize timezone-offset values to UTC at storage time +so all rows compare consistently. Timezone-naive datetimes interpreted as UTC +(document this). -Both types preserve the original JSON string at the serde boundary; conversion happens only when writing parquet. +Both types preserve the original JSON string at the serde boundary; conversion +happens only when writing parquet. -**`--where` queries:** Native date arithmetic works after this change. `WHERE date_part('year', published) = 2024`, `WHERE created > '2024-01-01'`, `WHERE updated BETWEEN '2024-06-01' AND '2024-06-30'`. +**`--where` queries:** Native date arithmetic works after this change. +`WHERE date_part('year', published) = 2024`, `WHERE created > '2024-01-01'`, +`WHERE updated BETWEEN '2024-06-01' AND '2024-06-30'`. ## Dependencies -- **`chrono`** (probably already transitively pulled in; if not, add explicitly). For string-to-Date32 parsing. `chrono::NaiveDate` + `chrono::DateTime`. No tz database needed (rely on offset strings in RFC 3339). +- **`chrono`** (probably already transitively pulled in; if not, add + explicitly). For string-to-Date32 parsing. `chrono::NaiveDate` + + `chrono::DateTime`. No tz database needed (rely on offset strings in RFC + 3339). - **`jsonschema`** already pulled; just enable `format` validation. ## Implementation waves ### Wave 1 — Date type (core) -Goal: `type = "Date"` works end-to-end (parse, validate, store, query). DateTime not yet in scope. - -- `crates/mdvs/src/discover/field_type.rs`: new `FieldType::Date` variant. Update `From`/`TryFrom` between FieldType and FieldTypeSerde. Update display chain. -- `crates/mdvs/src/schema/shared.rs`: extend parser to accept `Date` as a Scalar. Add `parse_scalar_date` test. -- `crates/mdvs/src/schema/json_schema.rs`: `dsl_to_canonical` emits `{"type": "string", "format": "date"}` for Date. `canonical_to_dsl` reverses. Gate updates to allow `format: date`. -- `crates/mdvs/src/cmd/check.rs`: enable format validation in the jsonschema Validator builder. -- `crates/mdvs/src/index/storage.rs`: handle Date in `build_array` — parse string → Date32. Update Arrow schema for Date columns. -- `crates/mdvs/src/discover/infer/types.rs`: pattern detection in widening. `Date + Date → Date`, `Date + String → String`, etc. -- Tests: unit (parse, display, widening), integration (init → check → build on a Date-typed field). +Goal: `type = "Date"` works end-to-end (parse, validate, store, query). DateTime +not yet in scope. + +- `crates/mdvs/src/discover/field_type.rs`: new `FieldType::Date` variant. + Update `From`/`TryFrom` between FieldType and FieldTypeSerde. Update display + chain. +- `crates/mdvs/src/schema/shared.rs`: extend parser to accept `Date` as a + Scalar. Add `parse_scalar_date` test. +- `crates/mdvs/src/schema/json_schema.rs`: `dsl_to_canonical` emits + `{"type": "string", "format": "date"}` for Date. `canonical_to_dsl` reverses. + Gate updates to allow `format: date`. +- `crates/mdvs/src/cmd/check.rs`: enable format validation in the jsonschema + Validator builder. +- `crates/mdvs/src/index/storage.rs`: handle Date in `build_array` — parse + string → Date32. Update Arrow schema for Date columns. +- `crates/mdvs/src/discover/infer/types.rs`: pattern detection in widening. + `Date + Date → Date`, `Date + String → String`, etc. +- Tests: unit (parse, display, widening), integration (init → check → build on a + Date-typed field). ### Wave 2 — Date inference Goal: `mdvs init` auto-detects Date fields without manual config. -- Pattern recognition in `infer_field_types`: if all observed strings for a field match `YYYY-MM-DD` with valid date ranges, classify as Date. +- Pattern recognition in `infer_field_types`: if all observed strings for a + field match `YYYY-MM-DD` with valid date ranges, classify as Date. - Strict semantics: any non-matching observation downgrades to String. -- example_kb effect: existing `date:` fields in meetings + experiments + blog get inferred as Date instead of String. -- Tests: pure-date observations → Date; mixed observations → String; edge cases (invalid month/day, leading zeros). +- example_kb effect: existing `date:` fields in meetings + experiments + blog + get inferred as Date instead of String. +- Tests: pure-date observations → Date; mixed observations → String; edge cases + (invalid month/day, leading zeros). ### Wave 3 — DateTime type (core + inference) -Goal: `type = "DateTime"` works end-to-end. Wave 1 + 2 patterns repeated for DateTime. +Goal: `type = "DateTime"` works end-to-end. Wave 1 + 2 patterns repeated for +DateTime. -- All Wave 1 changes parallel for DateTime, with RFC 3339 datetime regex + `Timestamp(Millisecond, Some("UTC"))` Arrow type. -- Timezone strategy: normalize to UTC at storage time. Document the rule for timezone-naive inputs (interpret as UTC). -- Inference: pattern recognition (`YYYY-MM-DDTHH:MM:SS[...]` with valid components). +- All Wave 1 changes parallel for DateTime, with RFC 3339 datetime regex + + `Timestamp(Millisecond, Some("UTC"))` Arrow type. +- Timezone strategy: normalize to UTC at storage time. Document the rule for + timezone-naive inputs (interpret as UTC). +- Inference: pattern recognition (`YYYY-MM-DDTHH:MM:SS[...]` with valid + components). - Widening: `Date + DateTime → String` (strict; no automatic promotion). - Tests: parse, display, widening, storage round-trip, timezone normalization. ### Wave 4 — Docs + example_kb -- `book/src/concepts/types.md`: add Date + DateTime to the supported-types table, update widening matrix, add a worked example using example_kb's `date` field. +- `book/src/concepts/types.md`: add Date + DateTime to the supported-types + table, update widening matrix, add a worked example using example_kb's `date` + field. - `book/src/configuration.md`: function-style grammar update. -- `book/src/concepts/validation.md`: mention `format: date` / `format: date-time` validation. +- `book/src/concepts/validation.md`: mention `format: date` / + `format: date-time` validation. - `docs/spec/architecture.md`: type system section. -- `docs/spec/storage.md`: Arrow type mappings table extended with Date32 + Timestamp. -- example_kb regen via `mdvs init --force` to demonstrate auto-inferred Date fields. +- `docs/spec/storage.md`: Arrow type mappings table extended with Date32 + + Timestamp. +- example_kb regen via `mdvs init --force` to demonstrate auto-inferred Date + fields. ## Out of scope (follow-ups) - `Time` (standalone time-of-day) — rare; add when asked. -- `min` / `max` constraints on Date / DateTime — needs `formatMinimum` vocab or Rust-side precheck. Track separately. -- Non-RFC-3339 format auto-detection (`MM/DD/YYYY`, `Jan 15, 2024`, the looser ISO 8601 variants, etc.) — `pattern` workaround for now. -- Obsidian wiki-link dates like `"[[2026-02-22]]"` — requires a Stage 1 field-value preprocessor; tracked via TODO-0009 / TODO-0144. -- Timezone-aware queries (`AT TIME ZONE`) — DataFusion supports it, but the v1 storage normalizes to UTC. Revisit if users want local-time queries. +- `min` / `max` constraints on Date / DateTime — needs `formatMinimum` vocab or + Rust-side precheck. Track separately. +- Non-RFC-3339 format auto-detection (`MM/DD/YYYY`, `Jan 15, 2024`, the looser + ISO 8601 variants, etc.) — `pattern` workaround for now. +- Obsidian wiki-link dates like `"[[2026-02-22]]"` — requires a Stage 1 + field-value preprocessor; tracked via TODO-0009 / TODO-0144. +- Timezone-aware queries (`AT TIME ZONE`) — DataFusion supports it, but the v1 + storage normalizes to UTC. Revisit if users want local-time queries. ## Verification 1. `cargo clippy --all-targets -p mdvs` clean. 2. `cargo fmt`. 3. `cargo test -p mdvs` green (~30-40 new tests across waves). -4. `mdvs init --force example_kb` produces a mdvs.toml where date fields are typed `Date` (meetings/blog dates) or `DateTime` (any if present). +4. `mdvs init --force example_kb` produces a mdvs.toml where date fields are + typed `Date` (meetings/blog dates) or `DateTime` (any if present). 5. `mdvs check example_kb` zero violations. -6. `mdvs build example_kb && mdvs search "..." example_kb --where 'date > "2024-01-01"'` returns only files after that date. +6. `mdvs build example_kb && mdvs search "..." example_kb --where 'date > "2024-01-01"'` + returns only files after that date. 7. mdbook build clean; widening matrix renders with Date + DateTime rows. ## Definition of done -- `Date` and `DateTime` fully integrated through parser, jsonschema, storage, and inference. +- `Date` and `DateTime` fully integrated through parser, jsonschema, storage, + and inference. - example_kb's existing date fields inferred as Date without manual editing. -- `--where` queries with date comparisons work natively (Date32 / Timestamp columns in parquet). +- `--where` queries with date comparisons work natively (Date32 / Timestamp + columns in parquet). - Specs + mdBook describe both types. - Memory updated with launch-criterion-1 marked done. diff --git a/docs/spec/todos/TODO-0008.md b/docs/spec/todos/TODO-0008.md index 5fcdf26..aa498b2 100644 --- a/docs/spec/todos/TODO-0008.md +++ b/docs/spec/todos/TODO-0008.md @@ -14,8 +14,14 @@ subsumed_by: 149 ## Original Scope -Add `min` and `max` constraints for range validation during check. Supports Integer, Float, Array(Integer), and Array(Float) fields. Bounds are inclusive. Optionally inferred via `update reinfer --with=range`. +Add `min` and `max` constraints for range validation during check. Supports +Integer, Float, Array(Integer), and Array(Float) fields. Bounds are inclusive. +Optionally inferred via `update reinfer --with=range`. ## Resolution -Partially implemented (Wave 1 — core `min`/`max` validation, `Range` variant, config validation, inference). The remaining validation code is subsumed by [TODO-0149](TODO-0149.md) — `validate_value` for range constraints will be delegated to `jsonschema` via `minimum`/`maximum` keywords. The serde layer, `validate_for_type`, `conflicts_with`, and inference logic remain. +Partially implemented (Wave 1 — core `min`/`max` validation, `Range` variant, +config validation, inference). The remaining validation code is subsumed by +[TODO-0149](TODO-0149.md) — `validate_value` for range constraints will be +delegated to `jsonschema` via `minimum`/`maximum` keywords. The serde layer, +`validate_for_type`, `conflicts_with`, and inference logic remain. diff --git a/docs/spec/todos/TODO-0009.md b/docs/spec/todos/TODO-0009.md index 792c2ba..5edb0b4 100644 --- a/docs/spec/todos/TODO-0009.md +++ b/docs/spec/todos/TODO-0009.md @@ -12,13 +12,18 @@ blocks: [] ## Summary -Allow users to define preprocessing pipelines that transform frontmatter values and/or file content before validation and storage. +Allow users to define preprocessing pipelines that transform frontmatter values +and/or file content before validation and storage. ## Details -**Motivation:** Obsidian vaults use conventions like wiki-link dates (`"[[2026-02-22]]"`) and wiki-link references (`"[[Collaborative IDE]]"`) in frontmatter. These wrapping characters should be stripped before validation (e.g. date parsing) and potentially before storage/search. +**Motivation:** Obsidian vaults use conventions like wiki-link dates +(`"[[2026-02-22]]"`) and wiki-link references (`"[[Collaborative IDE]]"`) in +frontmatter. These wrapping characters should be stripped before validation +(e.g. date parsing) and potentially before storage/search. **Possible use cases:** + - Strip `[[]]` from frontmatter values (wiki-link unwrapping) - Normalize date formats (e.g. `DD/MM/YYYY` → `YYYY-MM-DD`) - Lowercase tags for consistency @@ -26,13 +31,19 @@ Allow users to define preprocessing pipelines that transform frontmatter values - Custom content preprocessing before chunking/embedding **Design questions:** + - Where to define processors? In `mdvs.toml` per-field, or as a global pipeline? -- Built-in processors (strip-wikilinks, normalize-dates) vs user-defined (shell commands, regex)? -- At what stage do processors run? Before validation only, or also before storage? +- Built-in processors (strip-wikilinks, normalize-dates) vs user-defined (shell + commands, regex)? +- At what stage do processors run? Before validation only, or also before + storage? - Should processors be reversible (store original, validate transformed)? **Relationship to other TODOs:** -- TODO-0007 (Date type): date inference would benefit from a strip-wikilinks processor + +- TODO-0007 (Date type): date inference would benefit from a strip-wikilinks + processor - TODO-0006 (Enums): enum inference on processed values would be more accurate -This is a significant feature that needs careful design discussion before implementation. +This is a significant feature that needs careful design discussion before +implementation. diff --git a/docs/spec/todos/TODO-0010.md b/docs/spec/todos/TODO-0010.md index b2cd0f1..0e4f4ad 100644 --- a/docs/spec/todos/TODO-0010.md +++ b/docs/spec/todos/TODO-0010.md @@ -14,8 +14,12 @@ subsumed_by: 149 ## Original Scope -Add `min_length` and `max_length` constraints to String and Array fields for length validation during check. +Add `min_length` and `max_length` constraints to String and Array fields for +length validation during check. ## Resolution -Subsumed by [TODO-0149](TODO-0149.md). Length constraints will be implemented as JSON Schema keywords (`minLength`/`maxLength` for strings, `minItems`/`maxItems` for arrays) with validation delegated to `jsonschema`. No hand-rolled validation code needed. +Subsumed by [TODO-0149](TODO-0149.md). Length constraints will be implemented as +JSON Schema keywords (`minLength`/`maxLength` for strings, `minItems`/`maxItems` +for arrays) with validation delegated to `jsonschema`. No hand-rolled validation +code needed. diff --git a/docs/spec/todos/TODO-0011.md b/docs/spec/todos/TODO-0011.md index 1f3186d..4997ea7 100644 --- a/docs/spec/todos/TODO-0011.md +++ b/docs/spec/todos/TODO-0011.md @@ -16,14 +16,20 @@ files_updated: ## Summary -Build currently does a full re-chunk + re-embed on every run. Implement incremental builds that only re-embed changed files using content hashes. +Build currently does a full re-chunk + re-embed on every run. Implement +incremental builds that only re-embed changed files using content hashes. ## Resolution -Implemented incremental build as the default behavior. Build uses `content_hash` from existing `files.parquet` to classify files as new/edited/unchanged/removed. Only new and edited files are chunked and embedded. Model loading is skipped entirely when no embedding is needed. `--force` triggers full rebuild. +Implemented incremental build as the default behavior. Build uses `content_hash` +from existing `files.parquet` to classify files as new/edited/unchanged/removed. +Only new and edited files are chunked and embedded. Model loading is skipped +entirely when no embedding is needed. `--force` triggers full rebuild. Key additions: -- `FileIndexEntry` struct and `read_file_index()` with column projection in `storage.rs` + +- `FileIndexEntry` struct and `read_file_index()` with column projection in + `storage.rs` - `read_chunk_rows()` for full chunk deserialization in `storage.rs` - `FileClassification`, `FileToEmbed`, `classify_files()` in `build.rs` - `embed_file()` helper in `build.rs` diff --git a/docs/spec/todos/TODO-0012.md b/docs/spec/todos/TODO-0012.md index 503eced..f51ea83 100644 --- a/docs/spec/todos/TODO-0012.md +++ b/docs/spec/todos/TODO-0012.md @@ -15,8 +15,14 @@ files_updated: [src/index/storage.rs, src/cmd/build.rs] ## Summary -Write build configuration (model, revision, chunk_size, glob, built_at) as native parquet key-value metadata on files.parquet. Add a reader helper to extract it. +Write build configuration (model, revision, chunk_size, glob, built_at) as +native parquet key-value metadata on files.parquet. Add a reader helper to +extract it. ## Resolution -Added `BuildMetadata` struct to `storage.rs` composing `EmbeddingModelConfig` and `ChunkingConfig`. Metadata stored as `mdvs.*` prefixed key-value pairs on the Arrow schema of `files.parquet` (the primary build manifest). Added `write_parquet_with_metadata()` and `read_build_metadata()` helpers. Build constructs and writes metadata on every build. +Added `BuildMetadata` struct to `storage.rs` composing `EmbeddingModelConfig` +and `ChunkingConfig`. Metadata stored as `mdvs.*` prefixed key-value pairs on +the Arrow schema of `files.parquet` (the primary build manifest). Added +`write_parquet_with_metadata()` and `read_build_metadata()` helpers. Build +constructs and writes metadata on every build. diff --git a/docs/spec/todos/TODO-0013.md b/docs/spec/todos/TODO-0013.md index eaba0e4..a847910 100644 --- a/docs/spec/todos/TODO-0013.md +++ b/docs/spec/todos/TODO-0013.md @@ -14,8 +14,13 @@ files_updated: [src/cmd/search.rs] ## Summary -Search must compare the model/revision in mdvs.toml against what's stored in parquet metadata. Hard error on mismatch — prevents silently comparing embeddings from different models. +Search must compare the model/revision in mdvs.toml against what's stored in +parquet metadata. Hard error on mismatch — prevents silently comparing +embeddings from different models. ## Resolution -Search reads `BuildMetadata` from `files.parquet` before loading the model. Uses `PartialEq` on `EmbeddingModelConfig` to compare — single comparison catches model name and/or revision changes. Hard error with descriptive message on mismatch. Old indexes without metadata proceed silently (backwards compat). +Search reads `BuildMetadata` from `files.parquet` before loading the model. Uses +`PartialEq` on `EmbeddingModelConfig` to compare — single comparison catches +model name and/or revision changes. Hard error with descriptive message on +mismatch. Old indexes without metadata proceed silently (backwards compat). diff --git a/docs/spec/todos/TODO-0014.md b/docs/spec/todos/TODO-0014.md index 486d7a3..b595e53 100644 --- a/docs/spec/todos/TODO-0014.md +++ b/docs/spec/todos/TODO-0014.md @@ -14,8 +14,14 @@ files_updated: [src/cmd/build.rs, src/schema/shared.rs] ## Summary -Build should compare mdvs.toml config against parquet metadata to detect manual edits (e.g. user changed model name directly in toml). Require `--force` to confirm, same as `--set-*` flags. +Build should compare mdvs.toml config against parquet metadata to detect manual +edits (e.g. user changed model name directly in toml). Require `--force` to +confirm, same as `--set-*` flags. ## Resolution -After filling missing config sections, build reads `BuildMetadata` from `files.parquet` and compares `EmbeddingModelConfig` and `ChunkingConfig` via `PartialEq`. Collects all mismatches into a single error message. Requires `--force` to proceed. First build (no parquets) never needs `--force`. Added `Clone` and `Eq` derives to config structs for composability. +After filling missing config sections, build reads `BuildMetadata` from +`files.parquet` and compares `EmbeddingModelConfig` and `ChunkingConfig` via +`PartialEq`. Collects all mismatches into a single error message. Requires +`--force` to proceed. First build (no parquets) never needs `--force`. Added +`Clone` and `Eq` derives to config structs for composability. diff --git a/docs/spec/todos/TODO-0015.md b/docs/spec/todos/TODO-0015.md index 2dc32af..5280a88 100644 --- a/docs/spec/todos/TODO-0015.md +++ b/docs/spec/todos/TODO-0015.md @@ -15,12 +15,21 @@ files_updated: [src/main.rs] ## Summary -Replace the `todo!()` stubs for info and clean commands with working implementations. +Replace the `todo!()` stubs for info and clean commands with working +implementations. ## Resolution -**Clean** (`src/cmd/clean.rs`): Deletes `.mdvs/` directory if it exists, prints what was removed to stderr. Does not touch `mdvs.toml`. No `CommandOutput` trait needed. +**Clean** (`src/cmd/clean.rs`): Deletes `.mdvs/` directory if it exists, prints +what was removed to stderr. Does not touch `mdvs.toml`. No `CommandOutput` trait +needed. -**Info** (`src/cmd/info.rs`): Reads `mdvs.toml`, scans file count, reads fields with constraints (`InfoField`), reads parquet metadata and row counts if index exists (`IndexInfo`). `config_match` uses `PartialEq` on `EmbeddingModelConfig` and `ChunkingConfig` to detect config drift. Implements `CommandOutput` trait with human format showing scan info, fields with constraints (required/allowed), ignored fields, and index status. Supports `--output json` via `Serialize`. +**Info** (`src/cmd/info.rs`): Reads `mdvs.toml`, scans file count, reads fields +with constraints (`InfoField`), reads parquet metadata and row counts if index +exists (`IndexInfo`). `config_match` uses `PartialEq` on `EmbeddingModelConfig` +and `ChunkingConfig` to detect config drift. Implements `CommandOutput` trait +with human format showing scan info, fields with constraints (required/allowed), +ignored fields, and index status. Supports `--output json` via `Serialize`. -**CLI** (`src/main.rs`): Added `path` argument to Clean and Info variants, wired dispatch. 174 tests passing, 0 clippy warnings. +**CLI** (`src/main.rs`): Added `path` argument to Clean and Info variants, wired +dispatch. 174 tests passing, 0 clippy warnings. diff --git a/docs/spec/todos/TODO-0016.md b/docs/spec/todos/TODO-0016.md index 732f1c3..d5cdf5d 100644 --- a/docs/spec/todos/TODO-0016.md +++ b/docs/spec/todos/TODO-0016.md @@ -38,147 +38,177 @@ files_updated: ## Resolution -Shipped in three waves on branch `docs/todo-0016-lance-lancedb-design`, merged in PR #40 (commit `361f309`, 2026-05-25). A follow-on `docs/lance-truth-pass` branch (this commit) closes Wave 3. +Shipped in three waves on branch `docs/todo-0016-lance-lancedb-design`, merged +in PR #40 (commit `361f309`, 2026-05-25). A follow-on `docs/lance-truth-pass` +branch (this commit) closes Wave 3. ### What landed -- **Wave 1 — storage + build.** `LanceBackend` written; `Backend` enum collapsed to a single `Lance` variant; full Parquet+DataFusion stack deleted. Single Lance dataset at `.mdvs/index.lance/`, one row per chunk, persisted `chunk_text`, build metadata baked into the table-level kv. -- **Wave 2 — native search.** `SearchMode {Semantic, Fulltext, Hybrid}` enum with `--mode` CLI flag (default `Hybrid` via LanceDB's RRF reranker). Per-mode score column resolution (`_distance` → `1 - d`, `_score`, `_relevance_score`). Best-chunk-per-file dedupe in Rust (`OVER_FETCH_FACTOR = 3`). FTS index built every build; cosine IVF-PQ gated at `VECTOR_INDEX_MIN_ROWS = 10_000`. Verbose snippet now reads from persisted `chunk_text`. -- **Wave 2 follow-ups.** Schema-aware `--where` translator (`translate_where_to_struct`) — prefixes bare frontmatter names with `data.`, leaves scalar function calls bare, protects `date '...'` / `timestamp '...'` literals, rejects `Array(Float)` field references (TODO-0159 mitigation). `--limit 0` short-circuit; hybrid empty-batch guard; date/timestamp literal protection in the lit regex. -- **Wave 3 — docs sweep.** README, AGENTS.md, SKILL.md, mdbook (10 pages), all spec docs rewritten or revised. Single source of truth across user-facing and dev surfaces. `mdbook build` clean; cargo green throughout. +- **Wave 1 — storage + build.** `LanceBackend` written; `Backend` enum collapsed + to a single `Lance` variant; full Parquet+DataFusion stack deleted. Single + Lance dataset at `.mdvs/index.lance/`, one row per chunk, persisted + `chunk_text`, build metadata baked into the table-level kv. +- **Wave 2 — native search.** `SearchMode {Semantic, Fulltext, Hybrid}` enum + with `--mode` CLI flag (default `Hybrid` via LanceDB's RRF reranker). Per-mode + score column resolution (`_distance` → `1 - d`, `_score`, `_relevance_score`). + Best-chunk-per-file dedupe in Rust (`OVER_FETCH_FACTOR = 3`). FTS index built + every build; cosine IVF-PQ gated at `VECTOR_INDEX_MIN_ROWS = 10_000`. Verbose + snippet now reads from persisted `chunk_text`. +- **Wave 2 follow-ups.** Schema-aware `--where` translator + (`translate_where_to_struct`) — prefixes bare frontmatter names with `data.`, + leaves scalar function calls bare, protects `date '...'` / `timestamp '...'` + literals, rejects `Array(Float)` field references (TODO-0159 mitigation). + `--limit 0` short-circuit; hybrid empty-batch guard; date/timestamp literal + protection in the lit regex. +- **Wave 3 — docs sweep.** README, AGENTS.md, SKILL.md, mdbook (10 pages), all + spec docs rewritten or revised. Single source of truth across user-facing and + dev surfaces. `mdbook build` clean; cargo green throughout. ### Spinoff TODOs -- **[TODO-0157](TODO-0157.md)** — incremental ANN index optimize (currently rebuild-on-each-build; deferred per Wave 2 plan). Open, medium priority. -- **[TODO-0158](TODO-0158.md)** — quoted / special-char field names in `--where` (LanceDB filter grammar limitation). Open, low priority. -- **[TODO-0159](TODO-0159.md)** — lance-encoding 6.0 panic on `List` filter / scan; mitigation in place via the translator's Array(Float) rejection. Closed 2026-05-25 after filing upstream as [lancedb#3446](https://github.com/lancedb/lancedb/issues/3446). Reproducer (90-line `rust-script` spike + 2 KB IPC blob) now lives inside that issue. The mitigation in `translate_where_to_struct` will need to be removed when upstream fixes the panic. +- **[TODO-0157](TODO-0157.md)** — incremental ANN index optimize (currently + rebuild-on-each-build; deferred per Wave 2 plan). Open, medium priority. +- **[TODO-0158](TODO-0158.md)** — quoted / special-char field names in `--where` + (LanceDB filter grammar limitation). Open, low priority. +- **[TODO-0159](TODO-0159.md)** — lance-encoding 6.0 panic on `List` + filter / scan; mitigation in place via the translator's Array(Float) + rejection. Closed 2026-05-25 after filing upstream as + [lancedb#3446](https://github.com/lancedb/lancedb/issues/3446). Reproducer + (90-line `rust-script` spike + 2 KB IPC blob) now lives inside that issue. The + mitigation in `translate_where_to_struct` will need to be removed when + upstream fixes the panic. ### Acceptance verification - `cargo build`, `cargo test`, `cargo clippy --all-targets` clean. -- `mdbook build book` clean (one HTML-in-table-cell warning fixed during the docs sweep). -- `mdvs check / build / search / info / export-jsonschema` exercised end-to-end on `example_kb`; `--mode semantic`, `--mode fulltext`, `--mode hybrid` all return sensible results. -- `--where` smoke-tested for scalar comparisons, scalar function calls, comments / semicolons (no injection), arithmetic, FTS-via-`chunk_text LIKE`, lifecycle (body edit → incremental re-embed → new token findable; file delete → removed). -- Version bump to v0.6.0 deferred — paired with the next release cut after a quick Refractions smoke. +- `mdbook build book` clean (one HTML-in-table-cell warning fixed during the + docs sweep). +- `mdvs check / build / search / info / export-jsonschema` exercised end-to-end + on `example_kb`; `--mode semantic`, `--mode fulltext`, `--mode hybrid` all + return sensible results. +- `--where` smoke-tested for scalar comparisons, scalar function calls, comments + / semicolons (no injection), arithmetic, FTS-via-`chunk_text LIKE`, lifecycle + (body edit → incremental re-embed → new token findable; file delete → + removed). +- Version bump to v0.6.0 deferred — paired with the next release cut after a + quick Refractions smoke. --- -The original design and wave plan is preserved below for historical context. Cross-references to specific waves remain in the design text; the **Resolution** section above is authoritative for "what shipped where." +The original design and wave plan is preserved below for historical context. +Cross-references to specific waves remain in the design text; the **Resolution** +section above is authoritative for "what shipped where." ## Summary -Swap the storage and search engine entirely from Parquet + DataFusion to -Lance + LanceDB. **Not** a parallel backend; the Parquet codepath is -deleted in the same change. Includes native ANN indexing (IVF-PQ), BM25 -full-text indexing, and **hybrid search (vector + BM25 + RRF) as the -default search mode**. New `--mode semantic|fulltext|hybrid` flag exposes -the three retrieval modes; default is `hybrid`. +Swap the storage and search engine entirely from Parquet + DataFusion to Lance + +LanceDB. **Not** a parallel backend; the Parquet codepath is deleted in the same +change. Includes native ANN indexing (IVF-PQ), BM25 full-text indexing, and +**hybrid search (vector + BM25 + RRF) as the default search mode**. New +`--mode semantic|fulltext|hybrid` flag exposes the three retrieval modes; +default is `hybrid`. -This is the third and final launch-readiness item for the v0.6 / v1.0 -release. +This is the third and final launch-readiness item for the v0.6 / v1.0 release. ## Validation status (2026-05-22) Five throwaway `rust-script` spikes in `scripts/test_lance_*.rs` were run against the real `lancedb` 0.29 crate (which pins arrow 58 and -`lance-index = "=6.0.0"`) **before** committing to the swap. All five -pass; no blocking unknowns remain for waves 1–2. Findings are folded into -the design below. Summary: - -| Spike | Proves | Result | -|---|---|---| -| `test_lance_roundtrip.rs` | Denormalized schema (3-level nested `data` Struct + `FixedSizeList`) write→reopen | PASS | -| `test_lance_vector_search.rs` | Cosine kNN matches in-script brute-force exactly; BYO `Vec` | PASS | -| `test_lance_where.rs` | 14 `--where` clause families, incl. nested access, Date/Timestamp literals, `array_has` | PASS | -| `test_lance_fts_hybrid.rs` | BM25 fulltext + implicit RRF hybrid | PASS | -| `test_lance_metadata.rs` | Seven `mdvs.*` keys round-trip via schema-baked metadata | PASS | +`lance-index = "=6.0.0"`) **before** committing to the swap. All five pass; no +blocking unknowns remain for waves 1–2. Findings are folded into the design +below. Summary: + +| Spike | Proves | Result | +| ----------------------------- | ------------------------------------------------------------------------------------------ | ------ | +| `test_lance_roundtrip.rs` | Denormalized schema (3-level nested `data` Struct + `FixedSizeList`) write→reopen | PASS | +| `test_lance_vector_search.rs` | Cosine kNN matches in-script brute-force exactly; BYO `Vec` | PASS | +| `test_lance_where.rs` | 14 `--where` clause families, incl. nested access, Date/Timestamp literals, `array_has` | PASS | +| `test_lance_fts_hybrid.rs` | BM25 fulltext + implicit RRF hybrid | PASS | +| `test_lance_metadata.rs` | Seven `mdvs.*` keys round-trip via schema-baked metadata | PASS | ## Goals -- **Offline + embeddable** — LanceDB OSS mode runs in-process, persists to - the local filesystem. No service, no daemon. Same posture as today. -- **No feature regressions** — every `--where` operator we support today - keeps working: comparison ops, `IN`, `BETWEEN`, `IS NULL`, `LIKE`, - `AND`/`OR`, `array_has(col, val)`, struct/dotted-leaf access, date and - timestamp literals. -- **Performance** — ANN replaces brute-force cosine for vector search at - scale; brute force remains automatic up to a few hundred thousand - vectors. Build pipeline remains incremental (content-hash diff). -- **Hybrid search** — vector + BM25 + RRF reranking in a single query path, - with model2vec embeddings supplied explicitly (bring-your-own-embedding - pattern). +- **Offline + embeddable** — LanceDB OSS mode runs in-process, persists to the + local filesystem. No service, no daemon. Same posture as today. +- **No feature regressions** — every `--where` operator we support today keeps + working: comparison ops, `IN`, `BETWEEN`, `IS NULL`, `LIKE`, `AND`/`OR`, + `array_has(col, val)`, struct/dotted-leaf access, date and timestamp literals. +- **Performance** — ANN replaces brute-force cosine for vector search at scale; + brute force remains automatic up to a few hundred thousand vectors. Build + pipeline remains incremental (content-hash diff). +- **Hybrid search** — vector + BM25 + RRF reranking in a single query path, with + model2vec embeddings supplied explicitly (bring-your-own-embedding pattern). ## Non-goals - **Migration story for existing indexes.** mdvs has no users yet; old - `.mdvs/files.parquet` directories will simply emit "rebuild required" - on first run. No grace period, no auto-migration. -- **Compile-time feature flag for choosing between backends.** Earlier - draft of this TODO proposed `--features lancedb` with `ParquetBackend` - as the default. Decision (2026-05-21): full swap, single backend. - Maintenance surface stays small, dependency footprint shrinks. -- **Custom rerankers** beyond RRF in v0.6. LanceDB ships RRF as the - default reranker; that's enough. Custom rerankers are a separate TODO - if anyone asks for them. + `.mdvs/files.parquet` directories will simply emit "rebuild required" on first + run. No grace period, no auto-migration. +- **Compile-time feature flag for choosing between backends.** Earlier draft of + this TODO proposed `--features lancedb` with `ParquetBackend` as the default. + Decision (2026-05-21): full swap, single backend. Maintenance surface stays + small, dependency footprint shrinks. +- **Custom rerankers** beyond RRF in v0.6. LanceDB ships RRF as the default + reranker; that's enough. Custom rerankers are a separate TODO if anyone asks + for them. ## Design decisions ### Path: denormalize into a single Lance table -LanceDB's query API is single-table-oriented (no JOIN primitive). Instead -of two tables joined at search time (the current `files.parquet` + -`chunks.parquet` shape), the index becomes **one Lance table** where -each row is one chunk **with the file's metadata duplicated inline**. +LanceDB's query API is single-table-oriented (no JOIN primitive). Instead of two +tables joined at search time (the current `files.parquet` + `chunks.parquet` +shape), the index becomes **one Lance table** where each row is one chunk **with +the file's metadata duplicated inline**. Storage cost is small in practice — Lance's columnar encoding -run-length-compresses the repeated `filepath` and frontmatter values for -chunks belonging to the same file. Embedding bytes dominate the on-disk -size regardless. Expected growth: 5–15% vs today's two-Parquet layout. +run-length-compresses the repeated `filepath` and frontmatter values for chunks +belonging to the same file. Embedding bytes dominate the on-disk size +regardless. Expected growth: 5–15% vs today's two-Parquet layout. Query benefits: + - The JOIN disappears from the search SQL. -- The custom `cosine_similarity` UDF disappears — LanceDB's vector - search is a built-in API call. -- "Best chunk per file" (today's `ROW_NUMBER() OVER (PARTITION BY ...)`) - is post-processed in Rust by grouping the top-K result rows by - `file_id`. +- The custom `cosine_similarity` UDF disappears — LanceDB's vector search is a + built-in API call. +- "Best chunk per file" (today's `ROW_NUMBER() OVER (PARTITION BY ...)`) is + post-processed in Rust by grouping the top-K result rows by `file_id`. ### Schema (single Lance table `index`) -| Column | Arrow type | Notes | -|---|---|---| -| `chunk_id` | `Utf8` | UUID, primary identity | -| `file_id` | `Utf8` | Stable across rebuilds for unchanged files | -| `chunk_index` | `Int32` | 0-based position within the file | -| `start_line` | `Int32` | 1-based | -| `end_line` | `Int32` | 1-based, inclusive | -| `chunk_text` | `Utf8` | Plain text of the chunk (needed for FTS index) | -| `embedding` | `FixedSizeList` | Vector | -| `filepath` | `Utf8` | Duplicated per chunk | -| `content_hash` | `Utf8` | Duplicated per chunk | -| `data` | `Struct{...}` | Nested frontmatter Struct, duplicated per chunk | -| `built_at` | `Timestamp(Microsecond, UTC)` | Duplicated per chunk | - -`chunk_text` is **new** in this iteration — required for the BM25 FTS -index. Today's chunks.parquet doesn't persist chunk text (it's -regenerated on demand). Lance keeps it. +| Column | Arrow type | Notes | +| -------------- | ----------------------------- | ----------------------------------------------- | +| `chunk_id` | `Utf8` | UUID, primary identity | +| `file_id` | `Utf8` | Stable across rebuilds for unchanged files | +| `chunk_index` | `Int32` | 0-based position within the file | +| `start_line` | `Int32` | 1-based | +| `end_line` | `Int32` | 1-based, inclusive | +| `chunk_text` | `Utf8` | Plain text of the chunk (needed for FTS index) | +| `embedding` | `FixedSizeList` | Vector | +| `filepath` | `Utf8` | Duplicated per chunk | +| `content_hash` | `Utf8` | Duplicated per chunk | +| `data` | `Struct{...}` | Nested frontmatter Struct, duplicated per chunk | +| `built_at` | `Timestamp(Microsecond, UTC)` | Duplicated per chunk | + +`chunk_text` is **new** in this iteration — required for the BM25 FTS index. +Today's chunks.parquet doesn't persist chunk text (it's regenerated on demand). +Lance keeps it. ### Build metadata **Baked into the Arrow schema's metadata at `create_table` time** via `Schema::new_with_metadata(fields, map)`, read back with -`table.schema().await?.metadata()`. Spike `test_lance_metadata.rs` -confirmed all seven keys round-trip. This path is preferred over +`table.schema().await?.metadata()`. Spike `test_lance_metadata.rs` confirmed all +seven keys round-trip. This path is preferred over `Table::as_native().replace_schema_metadata(...)` because that method is -deprecation-flagged **and** is REPLACE-all (a single-key update silently -drops the rest) — a non-issue here since `build` knows all keys and -writes them in one shot, but the bake-at-create path sidesteps it -entirely. +deprecation-flagged **and** is REPLACE-all (a single-key update silently drops +the rest) — a non-issue here since `build` knows all keys and writes them in one +shot, but the bake-at-create path sidesteps it entirely. -Keys are the same as today's Parquet kv-metadata: `mdvs.provider`, -`mdvs.model`, `mdvs.revision`, `mdvs.chunk_size`, `mdvs.glob`, -`mdvs.built_at`, `mdvs.schema_hash`. Schema-hash mismatch between current -config and stored metadata still requires `mdvs build --force`. +Keys are the same as today's Parquet kv-metadata: `mdvs.provider`, `mdvs.model`, +`mdvs.revision`, `mdvs.chunk_size`, `mdvs.glob`, `mdvs.built_at`, +`mdvs.schema_hash`. Schema-hash mismatch between current config and stored +metadata still requires `mdvs build --force`. ### Search modes @@ -188,57 +218,54 @@ New global flag `--mode ` on `mdvs search`, accepting one of: - `fulltext` (BM25-only) — `table.query().full_text_search(query).limit(k)` - `hybrid` (default) — vector + BM25 + RRF reranker -`hybrid` is the default. The CLI surface is otherwise unchanged: same -`--where`, same `--limit`, same `--output`. Output format (table or -JSON) is unchanged regardless of mode — RRF returns one ranked list. +`hybrid` is the default. The CLI surface is otherwise unchanged: same `--where`, +same `--limit`, same `--output`. Output format (table or JSON) is unchanged +regardless of mode — RRF returns one ranked list. ### `--where` clause translation -The translator is **much smaller than this TODO originally assumed.** -Spike `test_lance_where.rs` proved LanceDB's filter parser accepts -**plain dotted struct paths** — `data.calibration.baseline.wavelength > -800` works with no backticks (the backticked form also works but is not -required). So there is **no backtick-quoting pass.** What the translator -must still do: - -1. **Prepend the `data.` struct prefix for frontmatter fields.** Today - users write `--where "calibration.baseline.wavelength > 800"` with no - prefix, because the `files_v` view promoted the `data` Struct's - children to top-level columns. With the view gone, the real column is - the `data` struct, so the translator rewrites a frontmatter leaf path - to `data.calibration.baseline.wavelength`. -2. **Be schema-aware** (corrected — see "Collisions still exist" below). - The wave-1 stopgap translator used a hardcoded reserved-name list and - prefixed *everything else* with `data.`. That is too naive: it - silently shadows a frontmatter field named like an internal column. - The real translator takes the **frontmatter field-name set** (from - the config) and: prefixes identifiers that are frontmatter fields with - `data.`; leaves genuine internal columns (`file_id`, `filepath`, …) - top-level; and on a name that is *both*, disambiguates via - `internal_prefix`/aliases (or errors). - -**Collisions still exist — `internal_prefix`/aliases are NOT obsolete.** -An earlier draft assumed the `data.` namespacing made collisions -impossible. It doesn't: it relocates them from "SQL view ambiguity" to -"bare-name resolution in the translator." If a frontmatter field is named -`file_id`, the user's bare `file_id` must resolve to the frontmatter -field, not the internal column. The old engine *detects* this collision -and errors with guidance (`search.rs:191-215`); wave 2 ports that -detection into the translator rather than dropping it. So the -`search()` signature keeps `internal_prefix` + `aliases`, and the -translator gains the frontmatter field-name list as input. - -The translator sits next to the existing quote-balance check in -`cmd/search.rs`. User-facing `--where` surface is identical to today. - -**Operators — all confirmed working** by the spike: `=`, `<>`, `<`, `>`, -`<=`, `>=`, `AND`, `OR`, `IN`, `BETWEEN`, `IS NULL`, `IS NOT NULL`, -`LIKE`, `array_has`, `date 'YYYY-MM-DD'` literals, `timestamp -'YYYY-MM-DDTHH:MM:SSZ'` literals. The book's `search-guide.md` examples -are exercised verbatim as integration tests in wave 2. - -The filter is applied via `table.query().only_if(clause)` — a single -predicate string, exactly the shape `--where` already produces. +The translator is **much smaller than this TODO originally assumed.** Spike +`test_lance_where.rs` proved LanceDB's filter parser accepts **plain dotted +struct paths** — `data.calibration.baseline.wavelength > 800` works with no +backticks (the backticked form also works but is not required). So there is **no +backtick-quoting pass.** What the translator must still do: + +1. **Prepend the `data.` struct prefix for frontmatter fields.** Today users + write `--where "calibration.baseline.wavelength > 800"` with no prefix, + because the `files_v` view promoted the `data` Struct's children to top-level + columns. With the view gone, the real column is the `data` struct, so the + translator rewrites a frontmatter leaf path to + `data.calibration.baseline.wavelength`. +2. **Be schema-aware** (corrected — see "Collisions still exist" below). The + wave-1 stopgap translator used a hardcoded reserved-name list and prefixed + _everything else_ with `data.`. That is too naive: it silently shadows a + frontmatter field named like an internal column. The real translator takes + the **frontmatter field-name set** (from the config) and: prefixes + identifiers that are frontmatter fields with `data.`; leaves genuine internal + columns (`file_id`, `filepath`, …) top-level; and on a name that is _both_, + disambiguates via `internal_prefix`/aliases (or errors). + +**Collisions still exist — `internal_prefix`/aliases are NOT obsolete.** An +earlier draft assumed the `data.` namespacing made collisions impossible. It +doesn't: it relocates them from "SQL view ambiguity" to "bare-name resolution in +the translator." If a frontmatter field is named `file_id`, the user's bare +`file_id` must resolve to the frontmatter field, not the internal column. The +old engine _detects_ this collision and errors with guidance +(`search.rs:191-215`); wave 2 ports that detection into the translator rather +than dropping it. So the `search()` signature keeps `internal_prefix` + +`aliases`, and the translator gains the frontmatter field-name list as input. + +The translator sits next to the existing quote-balance check in `cmd/search.rs`. +User-facing `--where` surface is identical to today. + +**Operators — all confirmed working** by the spike: `=`, `<>`, `<`, `>`, `<=`, +`>=`, `AND`, `OR`, `IN`, `BETWEEN`, `IS NULL`, `IS NOT NULL`, `LIKE`, +`array_has`, `date 'YYYY-MM-DD'` literals, `timestamp 'YYYY-MM-DDTHH:MM:SSZ'` +literals. The book's `search-guide.md` examples are exercised verbatim as +integration tests in wave 2. + +The filter is applied via `table.query().only_if(clause)` — a single predicate +string, exactly the shape `--where` already produces. ### Crate / dependency changes @@ -252,32 +279,34 @@ Added: - `lancedb` (0.29 at spike time) - `lance-index = "=6.0.0"` — required for `FullTextSearchQuery` - (`lance_index::scalar::FullTextSearchQuery`); not re-exported from - `lancedb`. The exact-version pin matches what `lancedb` 0.29 depends - on; bumping `lancedb` will dictate this version. -- `lance` only if direct API access is needed beyond `lancedb`; - otherwise transitive. + (`lance_index::scalar::FullTextSearchQuery`); not re-exported from `lancedb`. + The exact-version pin matches what `lancedb` 0.29 depends on; bumping + `lancedb` will dictate this version. +- `lance` only if direct API access is needed beyond `lancedb`; otherwise + transitive. -`tokio` remains (LanceDB is async). `arrow` (58, to match `lancedb`) -stays for in-memory RecordBatch construction. +`tokio` remains (LanceDB is async). `arrow` (58, to match `lancedb`) stays for +in-memory RecordBatch construction. **Spike-confirmed API ergonomics for wave 1:** + - `create_table` wants `Box` — wrap a `RecordBatchIterator`, not a bare boxed iterator. - Queries return a `futures` stream of `RecordBatch`; collect with `TryStreamExt::try_collect`. -- FTS index: `table.create_index(&["chunk_text"], - Index::FTS(Default::default())).execute()`. Default tokenizer - (whitespace + lowercase, no stemming) is fine for English markdown. +- FTS index: + `table.create_index(&["chunk_text"], Index::FTS(Default::default())).execute()`. + Default tokenizer (whitespace + lowercase, no stemming) is fine for English + markdown. ## Architecture ### Backend dispatch — it's an enum, not a trait The current code (`index/backend.rs`) is **not** a trait — it's an enum -`Backend { Parquet(ParquetBackend) }` with inherent methods that match on -the variant. That is exactly the project's "enum dispatch, no `dyn -Trait`" rule. Wave 1 **replaces the variant**, not the dispatch model: +`Backend { Parquet(ParquetBackend) }` with inherent methods that match on the +variant. That is exactly the project's "enum dispatch, no `dyn Trait`" rule. +Wave 1 **replaces the variant**, not the dispatch model: - `Backend::Parquet(ParquetBackend)` → `Backend::Lance(LanceBackend)` - delete the `Backend::parquet()` constructor, add `Backend::lance(root)` @@ -300,88 +329,82 @@ async fn search( ) -> anyhow::Result>; ``` -`SearchHit` gains the chunk text (`chunk_text: Option`) since -it's now persisted in the table. +`SearchHit` gains the chunk text (`chunk_text: Option`) since it's now +persisted in the table. ### Async conversion (wave 1, mechanical but cross-cutting) LanceDB's API is async to the core (`connect().execute().await`, -`create_table().execute().await`, `query()…execute().await`) — there is -no sync façade. Today only `Backend::search` is async (DataFusion forced -it); the other seven methods are sync because Parquet I/O is sync. With -Lance they all return futures. The ripple is **broad but shallow** — -mostly adding `.await` — because every backend call already sits inside -an `async fn`: - -- **`backend.rs`** — seven methods gain `async`: `write_index`, - `read_metadata`, `read_file_index`, `read_chunk_rows`, - `embedding_dimension`, `stats`, `clean`. (`search` already async; - `exists` stays sync — implemented as `lance_dir().exists()`, no I/O; - the `Backend::lance(root)` constructor stays sync, just stores a path.) - Because dispatch is an **enum, not a trait**, these are plain inherent - `async fn`s — no `async-trait` crate, no `dyn` boxing, no +`create_table().execute().await`, `query()…execute().await`) — there is no sync +façade. Today only `Backend::search` is async (DataFusion forced it); the other +seven methods are sync because Parquet I/O is sync. With Lance they all return +futures. The ripple is **broad but shallow** — mostly adding `.await` — because +every backend call already sits inside an `async fn`: + +- **`backend.rs`** — seven methods gain `async`: `write_index`, `read_metadata`, + `read_file_index`, `read_chunk_rows`, `embedding_dimension`, `stats`, `clean`. + (`search` already async; `exists` stays sync — implemented as + `lance_dir().exists()`, no I/O; the `Backend::lance(root)` constructor stays + sync, just stores a path.) Because dispatch is an **enum, not a trait**, these + are plain inherent `async fn`s — no `async-trait` crate, no `dyn` boxing, no object-safety issue. `#[instrument]` works on async fns unchanged. -- **`build.rs`** — `run` is already `async`; add `.await` at the ~6 - backend call sites (≈ lines 460/474/485/628/726/878). No new async fn. -- **`info.rs`, `clean.rs`** — flip `pub fn run` → `pub async fn run`, - `.await` their backend calls, and add `.await` at the two `main.rs` - dispatch sites (info ≈ 402, clean ≈ 375). `main` is already - `#[tokio::main]`. -- **`search.rs`** — already `async`; `.await` the two calls that newly - became async (`read_metadata`/`stats`). -- **Tests** — `#[test]` → `#[tokio::test]` for any test hitting a - now-async method (most of `backend.rs`'s test module, a couple in +- **`build.rs`** — `run` is already `async`; add `.await` at the ~6 backend call + sites (≈ lines 460/474/485/628/726/878). No new async fn. +- **`info.rs`, `clean.rs`** — flip `pub fn run` → `pub async fn run`, `.await` + their backend calls, and add `.await` at the two `main.rs` dispatch sites + (info ≈ 402, clean ≈ 375). `main` is already `#[tokio::main]`. +- **`search.rs`** — already `async`; `.await` the two calls that newly became + async (`read_metadata`/`stats`). +- **Tests** — `#[test]` → `#[tokio::test]` for any test hitting a now-async + method (most of `backend.rs`'s test module, a couple in `build.rs`/`search.rs`). **Rejected alternative: `block_on` inside sync methods.** Calling `Handle::current().block_on(...)` from within the running `#[tokio::main]` -worker panics ("Cannot start a runtime from within a runtime"), and it -hides that these commands now do real async I/O. Thread `async` through -instead. +worker panics ("Cannot start a runtime from within a runtime"), and it hides +that these commands now do real async I/O. Thread `async` through instead. ### Build pipeline -`build` flow (`cmd/build.rs`) is unchanged at the boundary: scan + -validate + chunk + embed → emit `Vec` and `Vec` → -hand to backend. The backend: +`build` flow (`cmd/build.rs`) is unchanged at the boundary: scan + validate + +chunk + embed → emit `Vec` and `Vec` → hand to backend. The +backend: -1. Builds a single denormalized Arrow `RecordBatch` (joins file and - chunk rows in memory), with the seven `mdvs.*` metadata keys baked - into the schema via `Schema::new_with_metadata`. -2. **Drop + recreate** the table: `conn.create_table("index", reader)` - with overwrite mode (wave 1 decision — see below). -3. *(wave 2)* Creates the BM25 FTS index on `chunk_text` — **always** - (no training minimum). -4. *(wave 2)* Creates the **cosine** IVF-PQ vector index on `embedding`, - **only above a row threshold** — see below. +1. Builds a single denormalized Arrow `RecordBatch` (joins file and chunk rows + in memory), with the seven `mdvs.*` metadata keys baked into the schema via + `Schema::new_with_metadata`. +2. **Drop + recreate** the table: `conn.create_table("index", reader)` with + overwrite mode (wave 1 decision — see below). +3. _(wave 2)_ Creates the BM25 FTS index on `chunk_text` — **always** (no + training minimum). +4. _(wave 2)_ Creates the **cosine** IVF-PQ vector index on `embedding`, **only + above a row threshold** — see below. **Vector index: cosine, explicit, threshold-gated (wave 2 decision).** + - **Cosine, set explicitly.** `IvfPqIndexBuilder::default()` is `DistanceType::L2`; we override with `Index::IvfPq(IvfPqIndexBuilder::default().distance_type(DistanceType::Cosine))`. - L2 is wrong for our non-normalized model2vec embeddings (it lets - magnitude influence ranking; we want semantic direction), and it would - diverge from the cosine parity baseline wave 1 established. `Index::Auto` - is **rejected**: it hardcodes L2 *and* always picks IVF-PQ regardless of - size (`table.rs:1868`). -- **Threshold-gated.** IVF-PQ must *train* k-means centroids and 256-entry - PQ codebooks; on tiny corpora (example_kb's ~59 chunks) that fails or is - meaningless. LanceDB says exact flat search is fine to ~100k vectors. So - build the index only when chunk count ≥ a threshold (start at ~10k, - tunable; possibly surfaced in `[search]` later); below it, no vector - index → `.nearest_to` runs exact flat scan (what gave byte-identical - wave-1 parity). `num_partitions`/`num_sub_vectors` left `None` - (LanceDB auto-suggests). - -**Why drop + recreate, not append / merge_insert (wave 1 decision).** -The incremental build already reconstructs the *complete* file + chunk -set in memory before calling the backend — it reads existing rows, -retains the unchanged chunk rows (`build.rs` ~554), re-embeds only the -changed files, then hands the backend everything. So the backend always -receives the full picture; overwriting the table is correct and dead -simple. `merge_insert`/`delete` row-level upserts buy nothing here and -are not used. (Index `optimize` for incremental ANN is a separate -deferral — [TODO-0157](TODO-0157.md).) + L2 is wrong for our non-normalized model2vec embeddings (it lets magnitude + influence ranking; we want semantic direction), and it would diverge from the + cosine parity baseline wave 1 established. `Index::Auto` is **rejected**: it + hardcodes L2 _and_ always picks IVF-PQ regardless of size (`table.rs:1868`). +- **Threshold-gated.** IVF-PQ must _train_ k-means centroids and 256-entry PQ + codebooks; on tiny corpora (example_kb's ~59 chunks) that fails or is + meaningless. LanceDB says exact flat search is fine to ~100k vectors. So build + the index only when chunk count ≥ a threshold (start at ~10k, tunable; + possibly surfaced in `[search]` later); below it, no vector index → + `.nearest_to` runs exact flat scan (what gave byte-identical wave-1 parity). + `num_partitions`/`num_sub_vectors` left `None` (LanceDB auto-suggests). + +**Why drop + recreate, not append / merge_insert (wave 1 decision).** The +incremental build already reconstructs the _complete_ file + chunk set in memory +before calling the backend — it reads existing rows, retains the unchanged chunk +rows (`build.rs` ~554), re-embeds only the changed files, then hands the backend +everything. So the backend always receives the full picture; overwriting the +table is correct and dead simple. `merge_insert`/`delete` row-level upserts buy +nothing here and are not used. (Index `optimize` for incremental ANN is a +separate deferral — [TODO-0157](TODO-0157.md).) ### Search path @@ -401,25 +424,25 @@ table.query() Spike-confirmed API notes (`test_lance_vector_search.rs`, `test_lance_fts_hybrid.rs`): -- **Cosine must be set explicitly.** LanceDB defaults to L2 distance; - without `.distance_type(DistanceType::Cosine)` rankings silently - diverge from today's cosine semantics. The result carries a - `_distance` column; `SearchHit` similarity = `1 - _distance`. +- **Cosine must be set explicitly.** LanceDB defaults to L2 distance; without + `.distance_type(DistanceType::Cosine)` rankings silently diverge from today's + cosine semantics. The result carries a `_distance` column; `SearchHit` + similarity = `1 - _distance`. - **Hybrid is implicit.** Setting **both** `.nearest_to()` and - `.full_text_search()` auto-routes `execute()` through LanceDB's - hybrid path with the **default RRF reranker** — no explicit - `.rerank(...)` call needed. Hybrid results add a `_relevance_score` - column (the fused score) alongside `_score`. + `.full_text_search()` auto-routes `execute()` through LanceDB's hybrid path + with the **default RRF reranker** — no explicit `.rerank(...)` call needed. + Hybrid results add a `_relevance_score` column (the fused score) alongside + `_score`. - Modes map directly onto which clauses are set: - `semantic` → `nearest_to` + `distance_type` only - `fulltext` → `full_text_search` only - `hybrid` → both (default) -Result rows are grouped by `file_id` in Rust; the highest-scoring row -per file becomes the `SearchHit`. The `over_fetch_factor` (initially 3) -compensates for chunk-level dedupe — if we want N file-level hits and -files have ~K chunks each, we need roughly N×K chunk-level hits. -Wave 2 tuning task: pick a sane default and document the tradeoff. +Result rows are grouped by `file_id` in Rust; the highest-scoring row per file +becomes the `SearchHit`. The `over_fetch_factor` (initially 3) compensates for +chunk-level dedupe — if we want N file-level hits and files have ~K chunks each, +we need roughly N×K chunk-level hits. Wave 2 tuning task: pick a sane default +and document the tradeoff. The per-mode score column differs: semantic → `_distance` (similarity = `1 - _distance`); fulltext → `_score` (BM25); hybrid → `_relevance_score` @@ -427,14 +450,13 @@ The per-mode score column differs: semantic → `_distance` (similarity = ### Snippet = persisted `chunk_text` (single source of truth) -`SearchHit.chunk_text` is populated from the **persisted `chunk_text` -column**, not re-read from disk. Wave 1 left `cmd/search.rs::read_lines` -reading the *current* file at search time — a second source that drifts -from the index if the file changed after `build`. Returning the indexed -`chunk_text` is the single source of truth and shows exactly the plain -text that was embedded/matched. `read_lines` is **deleted** in wave 2. -(The snippet changes from a raw-markdown slice to the embedded plain -text — intended.) +`SearchHit.chunk_text` is populated from the **persisted `chunk_text` column**, +not re-read from disk. Wave 1 left `cmd/search.rs::read_lines` reading the +_current_ file at search time — a second source that drifts from the index if +the file changed after `build`. Returning the indexed `chunk_text` is the single +source of truth and shows exactly the plain text that was embedded/matched. +`read_lines` is **deleted** in wave 2. (The snippet changes from a raw-markdown +slice to the embedded plain text — intended.) ## Wave plan @@ -443,104 +465,96 @@ text — intended.) **Scope.** Replace the storage layer end-to-end. Search still runs via a temporary in-Rust brute-force cosine loop during wave 1 only. -- Add `lancedb = "0.29"` dep only. **No Arrow conflict** — datafusion - 53.1 already resolves arrow 58.3 (the `arrow 57` Cargo comment was - stale), which unifies with lancedb's arrow 58. datafusion, parquet - feature, `src/search.rs`, and `ParquetBackend` all **stay compiling - and dormant** through wave 1 (live comparison baseline); wave 2 - deletes them together. -- **`chunk_text` deferred to wave 2.** It's only needed by the FTS - index; wave 1's throwaway search is semantic-only. Adding the required - field now would force edits to ~13 `ChunkRow` literals in - `src/search.rs` — a file wave 2 deletes. Add the column in wave 2 when - those sites are already gone (rebuild-required between versions - regardless). -- Add `LanceBackend` as a **second** `Backend` enum variant; route all - command call sites to `Backend::lance(path)`. `ParquetBackend` stays - as the dormant variant (deleted in wave 2 with the full swap). -- `storage.rs`: add `build_index_batch(schema_fields, files, chunks)` - that denormalizes file + chunk rows into one RecordBatch, reusing - `transpose_to_storage_type` + `build_array` for the `data` Struct. - The Parquet-only builders/readers stay (still used by the dormant +- Add `lancedb = "0.29"` dep only. **No Arrow conflict** — datafusion 53.1 + already resolves arrow 58.3 (the `arrow 57` Cargo comment was stale), which + unifies with lancedb's arrow 58. datafusion, parquet feature, `src/search.rs`, + and `ParquetBackend` all **stay compiling and dormant** through wave 1 (live + comparison baseline); wave 2 deletes them together. +- **`chunk_text` deferred to wave 2.** It's only needed by the FTS index; wave + 1's throwaway search is semantic-only. Adding the required field now would + force edits to ~13 `ChunkRow` literals in `src/search.rs` — a file wave 2 + deletes. Add the column in wave 2 when those sites are already gone + (rebuild-required between versions regardless). +- Add `LanceBackend` as a **second** `Backend` enum variant; route all command + call sites to `Backend::lance(path)`. `ParquetBackend` stays as the dormant + variant (deleted in wave 2 with the full swap). +- `storage.rs`: add `build_index_batch(schema_fields, files, chunks)` that + denormalizes file + chunk rows into one RecordBatch, reusing + `transpose_to_storage_type` + `build_array` for the `data` Struct. The + Parquet-only builders/readers stay (still used by the dormant `ParquetBackend`); they're deleted in wave 2. - `LanceBackend::write_index`: bake metadata into the schema (`Schema::new_with_metadata`), drop + recreate the table. -- `read_metadata` (schema metadata), `read_file_index` (distinct - file rows), `read_chunk_rows` (project chunk cols; no `chunk_text` - until wave 2), `embedding_dimension` (from schema), `stats`, `clean`. - `exists` stays sync. +- `read_metadata` (schema metadata), `read_file_index` (distinct file rows), + `read_chunk_rows` (project chunk cols; no `chunk_text` until wave 2), + `embedding_dimension` (from schema), `stats`, `clean`. `exists` stays sync. - **Async conversion** of the seven backend methods + `info::run` / `clean::run` + `.await` at all call sites (see Architecture). - Temporary `search()`: pull rows, apply `--where` via Lance `only_if` - (spike-proven; gets filter parity a wave early), brute-force cosine in - Rust, group best-per-file, top-K. Throwaway; deleted in wave 2. -- Tests: port `backend.rs`'s test module to `LanceBackend` - (`#[tokio::test]`); parity on example_kb — `build`, `check`, `info`, - `clean` green. - -**Acceptance**: `cargo test -p mdvs` green; `cargo clippy --all-targets` -clean; `cargo run -- build example_kb` produces `.mdvs/index.lance/`; -`cargo run -- search 'experiment' example_kb` returns the same top-K as -the pre-swap baseline (within float tolerance); incremental rebuild -(touch one file) re-embeds only that file. + (spike-proven; gets filter parity a wave early), brute-force cosine in Rust, + group best-per-file, top-K. Throwaway; deleted in wave 2. +- Tests: port `backend.rs`'s test module to `LanceBackend` (`#[tokio::test]`); + parity on example_kb — `build`, `check`, `info`, `clean` green. + +**Acceptance**: `cargo test -p mdvs` green; `cargo clippy --all-targets` clean; +`cargo run -- build example_kb` produces `.mdvs/index.lance/`; +`cargo run -- search 'experiment' example_kb` returns the same top-K as the +pre-swap baseline (within float tolerance); incremental rebuild (touch one file) +re-embeds only that file. ### Wave 2 — Native search, ANN, hybrid, mode flag, where-translator -**Scope.** Stand up the native LanceDB search path and the new mode -surface. Delete the temporary brute-force loop and the last vestiges of -DataFusion. +**Scope.** Stand up the native LanceDB search path and the new mode surface. +Delete the temporary brute-force loop and the last vestiges of DataFusion. -- **`chunk_text` column** (deferred from wave 1): add `chunk_text: - String` to `ChunkRow`, populate from `extract_plain_text(plain_text)` - in `build.rs`, add to `build_index_batch`, read in `read_chunk_rows`. - Update the surviving `ChunkRow` literals (fewer once `src/search.rs` - is gone). +- **`chunk_text` column** (deferred from wave 1): add `chunk_text: String` to + `ChunkRow`, populate from `extract_plain_text(plain_text)` in `build.rs`, add + to `build_index_batch`, read in `read_chunk_rows`. Update the surviving + `ChunkRow` literals (fewer once `src/search.rs` is gone). - **`SearchMode { Semantic, Fulltext, Hybrid }`** enum (enum dispatch, - `Default = Hybrid`, `clap::ValueEnum`). Add `--mode` to the Search - subcommand; thread `query_text: &str` + `mode: SearchMode` through - `search::run` → `backend.search`. + `Default = Hybrid`, `clap::ValueEnum`). Add `--mode` to the Search subcommand; + thread `query_text: &str` + `mode: SearchMode` through `search::run` → + `backend.search`. - Replace `LanceBackend::search` with the native query path: set - `nearest_to(emb)?.distance_type(Cosine)` and/or - `full_text_search(query_text)` per mode; `only_if(translate(where))`; - `limit(limit * over_fetch_factor)`; dedupe best-chunk-per-file in Rust; - read the per-mode score column (`_distance` / `_score` / - `_relevance_score`). -- **Snippet from persisted `chunk_text`**, delete `read_lines` (single - source of truth — see Search path). -- **Schema-aware `--where` translator** replacing the wave-1 stopgap: - prefix frontmatter fields with `data.`, leave internal columns, - port the collision detection + `internal_prefix`/aliases handling from - `search.rs:191-215`. Keep `internal_prefix`/`aliases` in the - `search()` signature (they're used, not ignored). -- **Indexes in `write_index`**: always build BM25 FTS on `chunk_text`; - build cosine IVF-PQ on `embedding` only above the row threshold. + `nearest_to(emb)?.distance_type(Cosine)` and/or `full_text_search(query_text)` + per mode; `only_if(translate(where))`; `limit(limit * over_fetch_factor)`; + dedupe best-chunk-per-file in Rust; read the per-mode score column + (`_distance` / `_score` / `_relevance_score`). +- **Snippet from persisted `chunk_text`**, delete `read_lines` (single source of + truth — see Search path). +- **Schema-aware `--where` translator** replacing the wave-1 stopgap: prefix + frontmatter fields with `data.`, leave internal columns, port the collision + detection + `internal_prefix`/aliases handling from `search.rs:191-215`. Keep + `internal_prefix`/`aliases` in the `search()` signature (they're used, not + ignored). +- **Indexes in `write_index`**: always build BM25 FTS on `chunk_text`; build + cosine IVF-PQ on `embedding` only above the row threshold. - Delete `src/search.rs`, `ParquetBackend` + the `Parquet` variant + `Backend::parquet()`, and the Parquet-only `storage.rs` helpers. -- **Remove `datafusion` + `parquet` feature; add `arrow` (58) direct - dep; migrate every `datafusion::arrow::X` → `arrow::X`** across - `storage.rs`, `discover/field_type.rs`, `index/backend.rs`, - `cmd/build.rs` (mechanical — identical types, different path). -- Integration tests: every `--where` example from `book/` run against a - built index; `--mode` tests (semantic-only, fulltext-only, hybrid); +- **Remove `datafusion` + `parquet` feature; add `arrow` (58) direct dep; + migrate every `datafusion::arrow::X` → `arrow::X`** across `storage.rs`, + `discover/field_type.rs`, `index/backend.rs`, `cmd/build.rs` (mechanical — + identical types, different path). +- Integration tests: every `--where` example from `book/` run against a built + index; `--mode` tests (semantic-only, fulltext-only, hybrid); collision-detection test; expand the translator unit tests. -**Acceptance**: `cargo test -p mdvs` green; manual smoke on -example_kb runs all three modes; the `Cargo.lock` no longer mentions -`datafusion` or `parquet` as direct deps. +**Acceptance**: `cargo test -p mdvs` green; manual smoke on example_kb runs all +three modes; the `Cargo.lock` no longer mentions `datafusion` or `parquet` as +direct deps. ### Wave 3 — Docs, demo, polish, version bump **Scope.** Sweep the docs and shipping artifacts; bump the version. -- `book/src/`: update concept pages mentioning Parquet, DataFusion, or - brute force. Add a "search modes" page covering the `--mode` flag and - hybrid's behavior. Update `search-guide.md` with any clarifications. -- `README.md`: refresh the feature bullets (vector + FTS + RRF mention), - update the architecture sentence ("Lance + LanceDB" instead of - "DataFusion + Parquet"). -- `assets/demo.py`: add one optional command demonstrating - `--mode fulltext` (a keyword-style query) so the GIF shows the new - surface; re-render `demo.gif`. +- `book/src/`: update concept pages mentioning Parquet, DataFusion, or brute + force. Add a "search modes" page covering the `--mode` flag and hybrid's + behavior. Update `search-guide.md` with any clarifications. +- `README.md`: refresh the feature bullets (vector + FTS + RRF mention), update + the architecture sentence ("Lance + LanceDB" instead of "DataFusion + + Parquet"). +- `assets/demo.py`: add one optional command demonstrating `--mode fulltext` (a + keyword-style query) so the GIF shows the new surface; re-render `demo.gif`. - `docs/spec/architecture.md`: rewrite the storage and search sections. - `docs/spec/storage.md`: replace (currently a stub). - `docs/spec/shared.md`: update column constants and Arrow types if any @@ -549,43 +563,42 @@ example_kb runs all three modes; the `Cargo.lock` no longer mentions - Bump to v0.6.0 (breaking change: storage format). **Acceptance**: `mdbook build` clean; -`cargo run -- search 'how to' Refractions --mode hybrid` returns -sensible results; release pipeline (`bump.yml`) succeeds. +`cargo run -- search 'how to' Refractions --mode hybrid` returns sensible +results; release pipeline (`bump.yml`) succeeds. ## Open questions to settle during implementation -These are not blockers but need answers as the work progresses; capture -the decisions in commit messages or follow-up TODOs. +These are not blockers but need answers as the work progresses; capture the +decisions in commit messages or follow-up TODOs. -- **`over_fetch_factor` for chunk-to-file dedupe.** Pick a sane default - in wave 2. Probably 3–5; benchmark on example_kb and Refractions. -- **ANN index parameters.** IVF-PQ has `num_partitions` and - `num_sub_vectors`. LanceDB picks defaults based on dataset size; - verify those defaults work on KBs of 10–100k chunks. Index is rebuilt - on each `build` (Option A); incremental optimize is deferred to - [TODO-0157](TODO-0157.md). +- **`over_fetch_factor` for chunk-to-file dedupe.** Pick a sane default in + wave 2. Probably 3–5; benchmark on example_kb and Refractions. +- **ANN index parameters.** IVF-PQ has `num_partitions` and `num_sub_vectors`. + LanceDB picks defaults based on dataset size; verify those defaults work on + KBs of 10–100k chunks. Index is rebuilt on each `build` (Option A); + incremental optimize is deferred to [TODO-0157](TODO-0157.md). Resolved by the spikes (no longer open): -- ~~**FTS tokenizer.**~~ Confirmed: default tokenizer (whitespace + - lowercase) returns the expected keyword matches on English markdown. - Revisit only if Refractions search quality dips in wave 3. -- ~~**Lance table metadata API.**~~ Resolved: bake into the Arrow schema - at create time (`Schema::new_with_metadata`); see Build metadata. -- ~~**Async surfacing.**~~ No rough edges — queries are `futures` - streams, `try_collect`-ed; commands are already `tokio`-async. +- ~~**FTS tokenizer.**~~ Confirmed: default tokenizer (whitespace + lowercase) + returns the expected keyword matches on English markdown. Revisit only if + Refractions search quality dips in wave 3. +- ~~**Lance table metadata API.**~~ Resolved: bake into the Arrow schema at + create time (`Schema::new_with_metadata`); see Build metadata. +- ~~**Async surfacing.**~~ No rough edges — queries are `futures` streams, + `try_collect`-ed; commands are already `tokio`-async. ## References -- **`scripts/test_lance_*.rs`** — the five committed de-risking spikes - (see Validation status). The authoritative record of the confirmed - API shapes; re-run with `rust-script` if a `lancedb` bump is suspected - of changing behavior. +- **`scripts/test_lance_*.rs`** — the five committed de-risking spikes (see + Validation status). The authoritative record of the confirmed API shapes; + re-run with `rust-script` if a `lancedb` bump is suspected of changing + behavior. - [LanceDB Rust crate docs](https://docs.rs/lancedb/latest/lancedb/) - [LanceDB metadata filtering reference](https://docs.lancedb.com/search/filtering) - [LanceDB hybrid search docs](https://docs.lancedb.com/search/hybrid-search) - [Lance + DataFusion integration](https://lancedb.github.io/lance/integrations/datafusion/) - Codebase audit (committed by sub-agent, 2026-05-21): inventories every - DataFusion / Parquet / Arrow surface we depend on, with file paths - and line numbers. Lives in this session's transcript; if it needs to - be referenced again, re-run the audit. + DataFusion / Parquet / Arrow surface we depend on, with file paths and line + numbers. Lives in this session's transcript; if it needs to be referenced + again, re-run the audit. diff --git a/docs/spec/todos/TODO-0017.md b/docs/spec/todos/TODO-0017.md index ffb29cf..a83ebe0 100644 --- a/docs/spec/todos/TODO-0017.md +++ b/docs/spec/todos/TODO-0017.md @@ -12,11 +12,15 @@ blocks: [] ## Summary -Add Ollama as an embedding provider, allowing users to generate embeddings via a local Ollama instance instead of in-process static models. +Add Ollama as an embedding provider, allowing users to generate embeddings via a +local Ollama instance instead of in-process static models. ## Details -Ollama exposes an `/api/embed` endpoint that accepts text and returns embeddings. This enables using transformer-based models (e.g. `nomic-embed-text`, `mxbai-embed-large`) which produce higher-quality embeddings than static models, at the cost of requiring a running Ollama service. +Ollama exposes an `/api/embed` endpoint that accepts text and returns +embeddings. This enables using transformer-based models (e.g. +`nomic-embed-text`, `mxbai-embed-large`) which produce higher-quality embeddings +than static models, at the cost of requiring a running Ollama service. ### Configuration @@ -30,9 +34,14 @@ endpoint = "http://localhost:11434" # default Ollama endpoint ### Key considerations - **Async HTTP**: calls to Ollama are async, uses `reqwest` or similar -- **Batching**: Ollama's `/api/embed` accepts multiple texts in one request — use this for batch embedding during build -- **Dimension discovery**: embed a probe string on load to determine dimension, or parse model metadata -- **Error handling**: Ollama service may be down, model may not be pulled — clear error messages needed -- **No credentials**: Ollama runs locally, no auth needed (unless user configures a remote instance) -- **Speed**: slower than static models (transformer inference vs lookup table), but still local (no network latency to cloud) +- **Batching**: Ollama's `/api/embed` accepts multiple texts in one request — + use this for batch embedding during build +- **Dimension discovery**: embed a probe string on load to determine dimension, + or parse model metadata +- **Error handling**: Ollama service may be down, model may not be pulled — + clear error messages needed +- **No credentials**: Ollama runs locally, no auth needed (unless user + configures a remote instance) +- **Speed**: slower than static models (transformer inference vs lookup table), + but still local (no network latency to cloud) - **Model availability**: user must `ollama pull ` before `mdvs build` diff --git a/docs/spec/todos/TODO-0018.md b/docs/spec/todos/TODO-0018.md index 720d81b..79ae377 100644 --- a/docs/spec/todos/TODO-0018.md +++ b/docs/spec/todos/TODO-0018.md @@ -12,11 +12,16 @@ blocks: [] ## Summary -Add cloud-hosted embedding providers (Azure AI Foundry, AWS Bedrock) as alternatives to local models, enabling high-quality embeddings from large transformer models without local compute. +Add cloud-hosted embedding providers (Azure AI Foundry, AWS Bedrock) as +alternatives to local models, enabling high-quality embeddings from large +transformer models without local compute. ## Details -Cloud providers expose embedding APIs via authenticated HTTPS endpoints. This enables using state-of-the-art models (e.g. Cohere embed, Titan embeddings) that may outperform local static models, at the cost of API credentials, network latency, and per-token pricing. +Cloud providers expose embedding APIs via authenticated HTTPS endpoints. This +enables using state-of-the-art models (e.g. Cohere embed, Titan embeddings) that +may outperform local static models, at the cost of API credentials, network +latency, and per-token pricing. ### Configuration examples @@ -40,10 +45,17 @@ region = "us-east-1" ### Key considerations - **Async HTTP**: all calls are async -- **Authentication**: credentials via environment variables (never in `mdvs.toml`). Support standard env var conventions for each provider -- **Rate limits**: cloud APIs enforce rate limits and token quotas — need retry with exponential backoff -- **Cost**: per-token pricing means large vaults cost money. Consider a `--dry-run` or confirmation prompt showing estimated token count before build -- **Batching**: APIs accept batches but with token limits per request — need to chunk batch requests appropriately -- **Latency**: network round-trips make build slower than local. Progress reporting becomes more important -- **Vendor SDKs vs raw HTTP**: evaluate whether to use vendor SDKs (adds heavy dependencies) or raw `reqwest` calls (simpler, lighter) -- **Feature flags**: cloud providers could be behind compile-time feature flags to keep the default binary lightweight +- **Authentication**: credentials via environment variables (never in + `mdvs.toml`). Support standard env var conventions for each provider +- **Rate limits**: cloud APIs enforce rate limits and token quotas — need retry + with exponential backoff +- **Cost**: per-token pricing means large vaults cost money. Consider a + `--dry-run` or confirmation prompt showing estimated token count before build +- **Batching**: APIs accept batches but with token limits per request — need to + chunk batch requests appropriately +- **Latency**: network round-trips make build slower than local. Progress + reporting becomes more important +- **Vendor SDKs vs raw HTTP**: evaluate whether to use vendor SDKs (adds heavy + dependencies) or raw `reqwest` calls (simpler, lighter) +- **Feature flags**: cloud providers could be behind compile-time feature flags + to keep the default binary lightweight diff --git a/docs/spec/todos/TODO-0019.md b/docs/spec/todos/TODO-0019.md index 82a87c4..d55617c 100644 --- a/docs/spec/todos/TODO-0019.md +++ b/docs/spec/todos/TODO-0019.md @@ -12,11 +12,14 @@ blocks: [] ## Summary -Add a `--verbose` / `-v` global CLI flag that enables detailed progress output across all commands. +Add a `--verbose` / `-v` global CLI flag that enables detailed progress output +across all commands. ## Details - Global flag on the root clap command, same pattern as `--output` -- When enabled, commands print additional detail to stderr (e.g., per-file status during build, per-field info during check, timing) +- When enabled, commands print additional detail to stderr (e.g., per-file + status during build, per-field info during check, timing) - Default behavior (no flag) stays concise as it is today -- Particularly useful for incremental build (showing which files were unchanged/changed/new/deleted) +- Particularly useful for incremental build (showing which files were + unchanged/changed/new/deleted) diff --git a/docs/spec/todos/TODO-0020.md b/docs/spec/todos/TODO-0020.md index 8525721..e34d747 100644 --- a/docs/spec/todos/TODO-0020.md +++ b/docs/spec/todos/TODO-0020.md @@ -15,8 +15,11 @@ blocks: [] ## Summary -`cargo publish --dry-run` warns about missing metadata fields. Add all required and recommended fields so crates.io renders the package correctly. +`cargo publish --dry-run` warns about missing metadata fields. Add all required +and recommended fields so crates.io renders the package correctly. ## Resolution -Added to `[package]` in Cargo.toml: `description`, `license = "MIT"`, `repository`, `readme`, `keywords`, `categories`. `cargo publish --dry-run` produces zero warnings. +Added to `[package]` in Cargo.toml: `description`, `license = "MIT"`, +`repository`, `readme`, `keywords`, `categories`. `cargo publish --dry-run` +produces zero warnings. diff --git a/docs/spec/todos/TODO-0021.md b/docs/spec/todos/TODO-0021.md index 5d87575..029f83c 100644 --- a/docs/spec/todos/TODO-0021.md +++ b/docs/spec/todos/TODO-0021.md @@ -19,12 +19,18 @@ subsumed: ## Summary -Edition 2024 requires nightly Rust. Downgrade to 2021 so stable Rust users can `cargo install mdvs`. +Edition 2024 requires nightly Rust. Downgrade to 2021 so stable Rust users can +`cargo install mdvs`. ## Resolution -Changed `edition = "2024"` to `edition = "2021"` in Cargo.toml. Rewrote 2 let chain usages: -- `src/index/backend.rs` — three-part let chain → nested `if let` with early return +Changed `edition = "2024"` to `edition = "2021"` in Cargo.toml. Rewrote 2 let +chain usages: + +- `src/index/backend.rs` — three-part let chain → nested `if let` with early + return - `src/cmd/search.rs` — two-part let chain → nested `if let` -Also resolved TODO-0026: the 3 `collapsible_if` clippy warnings were clippy suggesting let chains (edition 2024 feature). On edition 2021, clippy no longer suggests them — zero warnings. +Also resolved TODO-0026: the 3 `collapsible_if` clippy warnings were clippy +suggesting let chains (edition 2024 feature). On edition 2021, clippy no longer +suggests them — zero warnings. diff --git a/docs/spec/todos/TODO-0022.md b/docs/spec/todos/TODO-0022.md index 23c0956..fc13020 100644 --- a/docs/spec/todos/TODO-0022.md +++ b/docs/spec/todos/TODO-0022.md @@ -15,10 +15,13 @@ blocks: [] ## Summary -The README needs several updates before it accurately represents the published crate. +The README needs several updates before it accurately represents the published +crate. ## Resolution 1. Added `clean` command to the commands table (was missing) 2. Added installation section (`cargo install mdvs` + build from source) -3. Fixed `--where` example — changed `--where "tags = 'rust'"` to `--where "data['draft'] = false"` to match the actual DataFusion bracket syntax for the Arrow Struct column +3. Fixed `--where` example — changed `--where "tags = 'rust'"` to + `--where "data['draft'] = false"` to match the actual DataFusion bracket + syntax for the Arrow Struct column diff --git a/docs/spec/todos/TODO-0024.md b/docs/spec/todos/TODO-0024.md index f0ee7f8..cbd04e1 100644 --- a/docs/spec/todos/TODO-0024.md +++ b/docs/spec/todos/TODO-0024.md @@ -12,4 +12,5 @@ blocks: [] ## Details -Generated automatically by `cog bump` from conventional commits. The `[changelog]` section in `cog.toml` configures output format with GitHub links. +Generated automatically by `cog bump` from conventional commits. The +`[changelog]` section in `cog.toml` configures output format with GitHub links. diff --git a/docs/spec/todos/TODO-0025.md b/docs/spec/todos/TODO-0025.md index 1fbb546..d904a3b 100644 --- a/docs/spec/todos/TODO-0025.md +++ b/docs/spec/todos/TODO-0025.md @@ -15,8 +15,11 @@ blocks: [] ## Summary -The crate packages 113 files (195KB compressed) including spec archives, TODO files, notebooks, scripts, and test fixtures that users don't need. +The crate packages 113 files (195KB compressed) including spec archives, TODO +files, notebooks, scripts, and test fixtures that users don't need. ## Resolution -Added `exclude = ["docs/", "tests/", "scripts/", "notebooks/", ".claude/", "CLAUDE.md"]` to `[package]` in Cargo.toml. Package reduced to 31 files, 83KB compressed. +Added +`exclude = ["docs/", "tests/", "scripts/", "notebooks/", ".claude/", "CLAUDE.md"]` +to `[package]` in Cargo.toml. Package reduced to 31 files, 83KB compressed. diff --git a/docs/spec/todos/TODO-0026.md b/docs/spec/todos/TODO-0026.md index 48720e4..d4d148d 100644 --- a/docs/spec/todos/TODO-0026.md +++ b/docs/spec/todos/TODO-0026.md @@ -14,8 +14,12 @@ blocks: [] ## Original Scope -3 pre-existing `collapsible_if` warnings from clippy. Collapse the nested `if` blocks into single `if ... &&` expressions for a clean `cargo clippy` before release. +3 pre-existing `collapsible_if` warnings from clippy. Collapse the nested `if` +blocks into single `if ... &&` expressions for a clean `cargo clippy` before +release. ## Resolution -Subsumed by [TODO-0021](TODO-0021.md). The warnings were clippy suggesting edition 2024 let chains. Downgrading to edition 2021 eliminated them — clippy no longer suggests the pattern. +Subsumed by [TODO-0021](TODO-0021.md). The warnings were clippy suggesting +edition 2024 let chains. Downgrading to edition 2021 eliminated them — clippy no +longer suggests the pattern. diff --git a/docs/spec/todos/TODO-0027.md b/docs/spec/todos/TODO-0027.md index 5222743..2f1dbf7 100644 --- a/docs/spec/todos/TODO-0027.md +++ b/docs/spec/todos/TODO-0027.md @@ -24,8 +24,16 @@ files_updated: ## Summary -Rename internal columns in `files.parquet` and `chunks.parquet` with a configurable prefix (default `_`) so that frontmatter fields can be promoted to top-level columns without name collisions. +Rename internal columns in `files.parquet` and `chunks.parquet` with a +configurable prefix (default `_`) so that frontmatter fields can be promoted to +top-level columns without name collisions. ## Resolution -Added `StorageConfig` with `internal_prefix` field (default `"_"`) in a new `[storage]` section (hidden by default). The `col(prefix, name)` helper in `storage.rs` applies the prefix to all internal column names. `BuildMetadata` stores the prefix; changing it requires `--force` rebuild. Reserved name validation in `init`/`update` blocks frontmatter fields that collide with prefixed internal columns. Backward compat: old parquets (missing `mdvs.internal_prefix` key) default to `""`, triggering mismatch detection. +Added `StorageConfig` with `internal_prefix` field (default `"_"`) in a new +`[storage]` section (hidden by default). The `col(prefix, name)` helper in +`storage.rs` applies the prefix to all internal column names. `BuildMetadata` +stores the prefix; changing it requires `--force` rebuild. Reserved name +validation in `init`/`update` blocks frontmatter fields that collide with +prefixed internal columns. Backward compat: old parquets (missing +`mdvs.internal_prefix` key) default to `""`, triggering mismatch detection. diff --git a/docs/spec/todos/TODO-0028.md b/docs/spec/todos/TODO-0028.md index a8f420a..528714c 100644 --- a/docs/spec/todos/TODO-0028.md +++ b/docs/spec/todos/TODO-0028.md @@ -20,11 +20,15 @@ files_updated: ## Summary -Let users write `--where "draft = false"` instead of `--where "data['draft'] = false"`. Frontmatter fields should be queryable as top-level column names. +Let users write `--where "draft = false"` instead of +`--where "data['draft'] = false"`. Frontmatter fields should be queryable as +top-level column names. ## Resolution -Implemented via a DataFusion view (`files_v`) created in `SearchContext::new()`. After registering the `files` parquet table, the code inspects the `_data` Struct column's children from the Arrow schema and generates: +Implemented via a DataFusion view (`files_v`) created in `SearchContext::new()`. +After registering the `files` parquet table, the code inspects the `_data` +Struct column's children from the Arrow schema and generates: ```sql CREATE VIEW files_v AS @@ -32,6 +36,8 @@ SELECT *, _data['title'] AS title, _data['draft'] AS draft, ... FROM files ``` -The search SQL joins against `files_v` instead of `files`. Bare field names resolve naturally. The old `_data['field']` bracket syntax still works via `SELECT *` for backward compatibility. +The search SQL joins against `files_v` instead of `files`. Bare field names +resolve naturally. The old `_data['field']` bracket syntax still works via +`SELECT *` for backward compatibility. Updated help text (`main.rs`), README, spec, and tests to use bare field names. diff --git a/docs/spec/todos/TODO-0029.md b/docs/spec/todos/TODO-0029.md index 0b7ac8a..e5a0932 100644 --- a/docs/spec/todos/TODO-0029.md +++ b/docs/spec/todos/TODO-0029.md @@ -13,15 +13,31 @@ blocks: [] ## Summary -Create a user-facing documentation site using mdBook, deployed to GitHub Pages. Uses `example_kb/` as the running example throughout. The README covers installation and quick start; the docs site covers full usage, configuration reference, and features in action. +Create a user-facing documentation site using mdBook, deployed to GitHub Pages. +Uses `example_kb/` as the running example throughout. The README covers +installation and quick start; the docs site covers full usage, configuration +reference, and features in action. ## Key decisions -- **Location:** `book/` at repo root (not `docs/book/`). `docs/` is implementor-facing specs; `book/` is user-facing documentation. Different audience, different purpose. -- **Running example:** All pages use `example_kb/` (the Prismatiq Lab fixture) for examples and output snippets. This keeps examples consistent and testable. -- **Concepts split into sub-pages:** `concepts.md` is a hub linking to 4 focused sub-pages: types & widening, schema inference, validation, search & indexing. Each topic is dense enough (widening matrix, inference algorithm, violation types) to warrant its own page with examples. `search-guide.md` still covers `--where` syntax and ranking together. -- **Query examples in depth:** `search-guide.md` must include detailed, runnable examples for every query pattern: scalar filters, array containment (`array_has`, `= ANY()`), array length, nested object bracket access, field names with special characters, and combined filters. All examples against `example_kb/` so readers can copy-paste and try them. See `scripts/test_array_queries.rs` for tested query patterns to draw from. -- **Writing conventions:** see the `book` skill (`.claude/skills/book/SKILL.md`). +- **Location:** `book/` at repo root (not `docs/book/`). `docs/` is + implementor-facing specs; `book/` is user-facing documentation. Different + audience, different purpose. +- **Running example:** All pages use `example_kb/` (the Prismatiq Lab fixture) + for examples and output snippets. This keeps examples consistent and testable. +- **Concepts split into sub-pages:** `concepts.md` is a hub linking to 4 focused + sub-pages: types & widening, schema inference, validation, search & indexing. + Each topic is dense enough (widening matrix, inference algorithm, violation + types) to warrant its own page with examples. `search-guide.md` still covers + `--where` syntax and ranking together. +- **Query examples in depth:** `search-guide.md` must include detailed, runnable + examples for every query pattern: scalar filters, array containment + (`array_has`, `= ANY()`), array length, nested object bracket access, field + names with special characters, and combined filters. All examples against + `example_kb/` so readers can copy-paste and try them. See + `scripts/test_array_queries.rs` for tested query patterns to draw from. +- **Writing conventions:** see the `book` skill + (`.claude/skills/book/SKILL.md`). ## Checklist @@ -34,19 +50,28 @@ Create a user-facing documentation site using mdBook, deployed to GitHub Pages. ### Tier 1 — Core (read first) -- [x] Write `introduction.md` — what mdvs is, the "DB of documents" pitch, database analogy -- [x] Write `getting-started.md` — install, clone example_kb, `init`, `search`. Captured output from real runs. -- [x] Write `concepts.md` — hub page linking to sub-pages, quick summary of each concept area -- [x] Write `concepts/types.md` — FieldType enum, type widening matrix, nullable, examples from example_kb -- [x] Write `concepts/schema.md` — scan → inference → globs, allowed/required, bare files, ignore list, field states -- [x] Write `concepts/validation.md` — violation types, check behavior, String as top type, exit codes -- [x] Write `concepts/search.md` — chunking, embedding models, incremental build, ranking, --where overview +- [x] Write `introduction.md` — what mdvs is, the "DB of documents" pitch, + database analogy +- [x] Write `getting-started.md` — install, clone example_kb, `init`, `search`. + Captured output from real runs. +- [x] Write `concepts.md` — hub page linking to sub-pages, quick summary of each + concept area +- [x] Write `concepts/types.md` — FieldType enum, type widening matrix, + nullable, examples from example_kb +- [x] Write `concepts/schema.md` — scan → inference → globs, allowed/required, + bare files, ignore list, field states +- [x] Write `concepts/validation.md` — violation types, check behavior, String + as top type, exit codes +- [x] Write `concepts/search.md` — chunking, embedding models, incremental + build, ranking, --where overview ### Tier 2 — Commands -Each command page includes both compact (default) and verbose (`-v`) output examples. +Each command page includes both compact (default) and verbose (`-v`) output +examples. -- [x] Write `commands/init.md` — synopsis, flags, behavior, examples with captured output +- [x] Write `commands/init.md` — synopsis, flags, behavior, examples with + captured output - [x] Write `commands/check.md` - [x] Write `commands/update.md` - [x] Write `commands/build.md` @@ -56,16 +81,22 @@ Each command page includes both compact (default) and verbose (`-v`) output exam ### Tier 3 — Reference -- [x] Write `configuration.md` — full `mdvs.toml` reference, all sections, all fields, defaults. Annotate using example_kb's toml. -- [x] Write `search-guide.md` — `--where` deep dive: scalar, array, nested object, special characters, ranking. All queries runnable against example_kb. Must cover field name quoting/escaping (spaces, single quotes, double quotes) — command pages link here for hints explanation. +- [x] Write `configuration.md` — full `mdvs.toml` reference, all sections, all + fields, defaults. Annotate using example_kb's toml. +- [x] Write `search-guide.md` — `--where` deep dive: scalar, array, nested + object, special characters, ranking. All queries runnable against + example_kb. Must cover field name quoting/escaping (spaces, single quotes, + double quotes) — command pages link here for hints explanation. ### Tier 4 — Recipes -- [x] Write `recipes/obsidian.md` — setting up mdvs on an Obsidian vault, `.mdvsignore`, typical patterns +- [x] Write `recipes/obsidian.md` — setting up mdvs on an Obsidian vault, + `.mdvsignore`, typical patterns - [ ] Write `recipes/ci.md` — deferred, needs testing first (TODO-0105) ### Finalize -- [ ] GitHub Actions workflow to build and deploy to GitHub Pages on push to main (see [TODO-0095](TODO-0095.md)) +- [ ] GitHub Actions workflow to build and deploy to GitHub Pages on push to + main (see [TODO-0095](TODO-0095.md)) - [x] Delete old book (`docs/book/`) - [x] Link from README: "Full documentation at ..." diff --git a/docs/spec/todos/TODO-0030.md b/docs/spec/todos/TODO-0030.md index a40d137..9147d3a 100644 --- a/docs/spec/todos/TODO-0030.md +++ b/docs/spec/todos/TODO-0030.md @@ -12,19 +12,25 @@ blocks: [] ## Summary -Publish mdvs via Homebrew and provide prebuilt binaries for macOS and Linux so users don't need the Rust toolchain to install. +Publish mdvs via Homebrew and provide prebuilt binaries for macOS and Linux so +users don't need the Rust toolchain to install. ## Details ### Steps -1. **GitHub Actions release workflow** — on tag push, build release binaries for macOS (aarch64 + x86_64) and Linux (x86_64, musl) -2. **cargo-dist or release-plz** — automates binary builds, GitHub Releases, Homebrew formula generation, and shell installer scripts -3. **Homebrew tap** — create `edochi/homebrew-tap` repo with a formula that downloads the prebuilt binary +1. **GitHub Actions release workflow** — on tag push, build release binaries for + macOS (aarch64 + x86_64) and Linux (x86_64, musl) +2. **cargo-dist or release-plz** — automates binary builds, GitHub Releases, + Homebrew formula generation, and shell installer scripts +3. **Homebrew tap** — create `edochi/homebrew-tap` repo with a formula that + downloads the prebuilt binary 4. **README install section** — add `brew install edochi/tap/mdvs` ### Notes -- `cargo-dist` is the Rust ecosystem standard for this — generates Homebrew formulae, shell installers, and GitHub releases in one CI step -- Prebuilt binaries are important because mdvs depends on `ort` (ONNX Runtime) which has a non-trivial build from source +- `cargo-dist` is the Rust ecosystem standard for this — generates Homebrew + formulae, shell installers, and GitHub releases in one CI step +- Prebuilt binaries are important because mdvs depends on `ort` (ONNX Runtime) + which has a non-trivial build from source - Consider also publishing to `crates.io` first (separate from this TODO) diff --git a/docs/spec/todos/TODO-0031.md b/docs/spec/todos/TODO-0031.md index afc3e33..0013c4c 100644 --- a/docs/spec/todos/TODO-0031.md +++ b/docs/spec/todos/TODO-0031.md @@ -13,21 +13,32 @@ blocks: [] ## Summary -Provide a synthetic markdown vault that users can use to try mdvs features hands-on. +Provide a synthetic markdown vault that users can use to try mdvs features +hands-on. ## Original Scope -Originally planned as a separate repository (`edochi/mdvs-example-vault`). Decided against it — a separate repo adds maintenance overhead (keeping in sync with CLI changes, separate CI) for marginal benefit. mdvs is a CLI tool that operates on existing files; users will run it on their own notes, not fork a starter repo. +Originally planned as a separate repository (`edochi/mdvs-example-vault`). +Decided against it — a separate repo adds maintenance overhead (keeping in sync +with CLI changes, separate CI) for marginal benefit. mdvs is a CLI tool that +operates on existing files; users will run it on their own notes, not fork a +starter repo. ## Resolution -Covered by the in-repo `example_kb/` directory (added in `8121794`). It's a synthetic research lab journal (Prismatiq) with 43 markdown files across 22 directories, exercising edge cases: bare files, null values, type widening, nested objects, special characters in field names, empty frontmatter, 5-level deep nesting, and varying field density. +Covered by the in-repo `example_kb/` directory (added in `8121794`). It's a +synthetic research lab journal (Prismatiq) with 43 markdown files across 22 +directories, exercising edge cases: bare files, null values, type widening, +nested objects, special characters in field names, empty frontmatter, 5-level +deep nesting, and varying field density. Users can try it with: + ```bash git clone https://github.com/edochi/mdvs.git mdvs init mdvs/example_kb mdvs search "experiment results" mdvs/example_kb ``` -Remaining: link `example_kb/` from the README and mdBook docs (can be done as part of TODO-0048 or TODO-0029). +Remaining: link `example_kb/` from the README and mdBook docs (can be done as +part of TODO-0048 or TODO-0029). diff --git a/docs/spec/todos/TODO-0032.md b/docs/spec/todos/TODO-0032.md index 2972727..b6b1cc2 100644 --- a/docs/spec/todos/TODO-0032.md +++ b/docs/spec/todos/TODO-0032.md @@ -12,20 +12,29 @@ blocks: [] ## Summary -The `-v`/`-vv`/`-vvv` flags produce almost no useful output. Span names appear without elapsed times, and there are very few informational log events. Verbose mode should show step timing and operational details. +The `-v`/`-vv`/`-vvv` flags produce almost no useful output. Span names appear +without elapsed times, and there are very few informational log events. Verbose +mode should show step timing and operational details. ## Details ### Current problems -1. **No timing on spans** — `tracing-tree` shows span names (e.g. `scan`, `validate`, `search_index`) but not how long each took. The elapsed time only appears sporadically. -2. **Almost no log events** — even at `-vvv` (trace level), very little is logged beyond span entry/exit. Missing: file counts, chunk counts, model load time, embed time, query time, files scanned, violations found, etc. -3. **Not useful for debugging** — a user running `-v` to understand why something is slow gets no actionable information. +1. **No timing on spans** — `tracing-tree` shows span names (e.g. `scan`, + `validate`, `search_index`) but not how long each took. The elapsed time only + appears sporadically. +2. **Almost no log events** — even at `-vvv` (trace level), very little is + logged beyond span entry/exit. Missing: file counts, chunk counts, model load + time, embed time, query time, files scanned, violations found, etc. +3. **Not useful for debugging** — a user running `-v` to understand why + something is slow gets no actionable information. ### Expected behavior -- `-v` (info): step names with elapsed time, key counts (files scanned, chunks embedded, results found) -- `-vv` (debug): per-file details (which files changed, which were skipped, embed times) +- `-v` (info): step names with elapsed time, key counts (files scanned, chunks + embedded, results found) +- `-vv` (debug): per-file details (which files changed, which were skipped, + embed times) - `-vvv` (trace): internal details (SQL queries, Arrow schema, model config) ### Files to investigate diff --git a/docs/spec/todos/TODO-0033.md b/docs/spec/todos/TODO-0033.md index 4cab293..5765aaa 100644 --- a/docs/spec/todos/TODO-0033.md +++ b/docs/spec/todos/TODO-0033.md @@ -21,7 +21,7 @@ Umbrella TODO — split into individual tasks: 7. **TODO-0040**: Rewrite init command output 8. **TODO-0041**: Rewrite update command output 9. **TODO-0042**: Rewrite info command output -10. **TODO-0043**: Fix tracing levels — distinct debug/trace events with elapsed times +10. **TODO-0043**: Fix tracing levels — distinct debug/trace events with elapsed + times -Design spec: `docs/spec/output-format.md` -Prototype: `scripts/test_tables.rs` +Design spec: `docs/spec/output-format.md` Prototype: `scripts/test_tables.rs` diff --git a/docs/spec/todos/TODO-0034.md b/docs/spec/todos/TODO-0034.md index 8ace7cc..b78f973 100644 --- a/docs/spec/todos/TODO-0034.md +++ b/docs/spec/todos/TODO-0034.md @@ -12,19 +12,23 @@ blocks: [35, 36, 37, 38, 39, 40, 41, 42] ## Summary -Rework CLI flags as prerequisite for the unified output format. All command rewrites depend on this. +Rework CLI flags as prerequisite for the unified output format. All command +rewrites depend on this. ## Changes - Rename `OutputFormat::Human` → `OutputFormat::Text` in clap + output trait -- Rename `format_human()` → `format_text()` in `CommandOutput` trait, add `verbose: bool` parameter +- Rename `format_human()` → `format_text()` in `CommandOutput` trait, add + `verbose: bool` parameter - Replace `-v`/`-vv`/`-vvv` with `--logs={info,debug,trace}` for stderr tracing - Add `-v` as boolean flag for output detail level -- Pass `verbose: bool` through to `format_text(verbose)` and to command `run()` functions +- Pass `verbose: bool` through to `format_text(verbose)` and to command `run()` + functions - Update `print()` dispatch in `CommandOutput` trait ## Files to modify - `src/main.rs` — CLI arg parsing, tracing subscriber setup - `src/output.rs` — `CommandOutput` trait, `OutputFormat` enum -- `src/cmd/*.rs` — update all `format_human()` signatures to `format_text(&self, verbose: bool)` +- `src/cmd/*.rs` — update all `format_human()` signatures to + `format_text(&self, verbose: bool)` diff --git a/docs/spec/todos/TODO-0035.md b/docs/spec/todos/TODO-0035.md index 8a52c3e..3a45399 100644 --- a/docs/spec/todos/TODO-0035.md +++ b/docs/spec/todos/TODO-0035.md @@ -12,14 +12,17 @@ blocks: [36, 37, 38, 39, 40, 41, 42] ## Summary -Add table rendering dependencies and create reusable style helpers for all command output. +Add table rendering dependencies and create reusable style helpers for all +command output. ## Changes - Add `tabled = "0.20"` and `terminal_size = "0.4"` to `Cargo.toml` - Create table helper module (in `src/output.rs` or `src/table.rs`) -- `style_compact()`: `Style::rounded()` + `remove_horizontals()` + `Width::increase(w)` + `Width::wrap(w)` -- `style_record(cols)`: `Style::rounded()` + `ColumnSpan` on detail row + `BorderCorrection` + `Width::increase(w)` + `Width::wrap(w)` +- `style_compact()`: `Style::rounded()` + `remove_horizontals()` + + `Width::increase(w)` + `Width::wrap(w)` +- `style_record(cols)`: `Style::rounded()` + `ColumnSpan` on detail row + + `BorderCorrection` + `Width::increase(w)` + `Width::wrap(w)` - `term_width()` helper using `terminal_size` crate (fallback 80) ## Prototype diff --git a/docs/spec/todos/TODO-0036.md b/docs/spec/todos/TODO-0036.md index d7f2df7..0382bdb 100644 --- a/docs/spec/todos/TODO-0036.md +++ b/docs/spec/todos/TODO-0036.md @@ -30,4 +30,5 @@ Cleaned ".mdvs" ## Files to modify -- `src/cmd/clean.rs` — `format_text()`, add `elapsed_ms` + file size to result struct +- `src/cmd/clean.rs` — `format_text()`, add `elapsed_ms` + file size to result + struct diff --git a/docs/spec/todos/TODO-0037.md b/docs/spec/todos/TODO-0037.md index 0941c22..0770eee 100644 --- a/docs/spec/todos/TODO-0037.md +++ b/docs/spec/todos/TODO-0037.md @@ -12,7 +12,8 @@ blocks: [] ## Summary -Compact: ranked hit table with index/path/score. Verbose: record tables with best chunk text per file read from disk via `start_line`/`end_line`. +Compact: ranked hit table with index/path/score. Verbose: record tables with +best chunk text per file read from disk via `start_line`/`end_line`. ## Compact @@ -40,11 +41,13 @@ Searched "rust" — 10 hits 10 hits | model: "minishlab/potion-base-8M" | limit: 10 | 580ms ``` -Known limitation: only the top chunk per file is shown. Limit applies to files, not chunks. +Known limitation: only the top chunk per file is shown. Limit applies to files, +not chunks. ## Changes -- Modify search SQL or add post-query to get chunk `start_line`/`end_line` for best chunk per file +- Modify search SQL or add post-query to get chunk `start_line`/`end_line` for + best chunk per file - Read chunk text from original `.md` files at search time - Add `elapsed_ms`, model info to result struct diff --git a/docs/spec/todos/TODO-0038.md b/docs/spec/todos/TODO-0038.md index f5347bb..7437c9c 100644 --- a/docs/spec/todos/TODO-0038.md +++ b/docs/spec/todos/TODO-0038.md @@ -12,7 +12,8 @@ blocks: [] ## Summary -Compact: stats table (embedded/unchanged/removed with files + chunks). Verbose: record tables with per-file lists. +Compact: stats table (embedded/unchanged/removed with files + chunks). Verbose: +record tables with per-file lists. ## Compact (incremental) @@ -38,7 +39,8 @@ Built index — 498 files, 2314 chunks (full rebuild) ## Verbose (incremental) -Record tables with file lists under embedded/removed, footer with model + glob + elapsed. +Record tables with file lists under embedded/removed, footer with model + glob + +elapsed. ## Changes @@ -48,8 +50,13 @@ Record tables with file lists under embedded/removed, footer with model + glob + ## Output alignment rule -Text and JSON must contain the same data — only the format differs. Verbose-only detail fields are `Option` with `#[serde(skip_serializing_if = "Option::is_none")]`, populated by `run(verbose: bool)`. Both `format_text()` and JSON serialization use the same struct. +Text and JSON must contain the same data — only the format differs. Verbose-only +detail fields are `Option` with +`#[serde(skip_serializing_if = "Option::is_none")]`, populated by +`run(verbose: bool)`. Both `format_text()` and JSON serialization use the same +struct. ## Files to modify -- `src/cmd/build.rs` — `format_text()`, result struct enrichment, `run()` takes `verbose` +- `src/cmd/build.rs` — `format_text()`, result struct enrichment, `run()` takes + `verbose` diff --git a/docs/spec/todos/TODO-0039.md b/docs/spec/todos/TODO-0039.md index 1117bd5..2441c06 100644 --- a/docs/spec/todos/TODO-0039.md +++ b/docs/spec/todos/TODO-0039.md @@ -12,7 +12,9 @@ blocks: [] ## Summary -Compact: violation table (field/kind/count) + new fields table. Verbose: record tables with file lists per violation. Handles four cases: no violations, violations only, new fields only, both. +Compact: violation table (field/kind/count) + new fields table. Verbose: record +tables with file lists per violation. Handles four cases: no violations, +violations only, new fields only, both. ## Compact (with violations) @@ -28,12 +30,18 @@ Checked 498 files — 3 violation(s) ## Verbose (with violations) -Record tables with file lists (and type detail for WrongType), footer with glob + elapsed. +Record tables with file lists (and type detail for WrongType), footer with +glob + elapsed. ## Output alignment rule -Text and JSON must contain the same data — only the format differs. Verbose-only detail fields are `Option` with `#[serde(skip_serializing_if = "Option::is_none")]`, populated by `run(verbose: bool)`. Both `format_text()` and JSON serialization use the same struct. +Text and JSON must contain the same data — only the format differs. Verbose-only +detail fields are `Option` with +`#[serde(skip_serializing_if = "Option::is_none")]`, populated by +`run(verbose: bool)`. Both `format_text()` and JSON serialization use the same +struct. ## Files to modify -- `src/cmd/check.rs` — `format_text()`, result struct enrichment, `run()` takes `verbose` +- `src/cmd/check.rs` — `format_text()`, result struct enrichment, `run()` takes + `verbose` diff --git a/docs/spec/todos/TODO-0040.md b/docs/spec/todos/TODO-0040.md index bc752fa..5741048 100644 --- a/docs/spec/todos/TODO-0040.md +++ b/docs/spec/todos/TODO-0040.md @@ -12,7 +12,9 @@ blocks: [] ## Summary -Compact: field table (name/type/count). Verbose: record tables with required/allowed per field, footer with glob + elapsed. Dry run appends `(dry run)` to one-liner. +Compact: field table (name/type/count). Verbose: record tables with +required/allowed per field, footer with glob + elapsed. Dry run appends +`(dry run)` to one-liner. ## Compact @@ -32,8 +34,13 @@ Record tables with required/allowed glob lists per field. ## Output alignment rule -Text and JSON must contain the same data — only the format differs. Verbose-only detail fields are `Option` with `#[serde(skip_serializing_if = "Option::is_none")]`, populated by `run(verbose: bool)`. Both `format_text()` and JSON serialization use the same struct. +Text and JSON must contain the same data — only the format differs. Verbose-only +detail fields are `Option` with +`#[serde(skip_serializing_if = "Option::is_none")]`, populated by +`run(verbose: bool)`. Both `format_text()` and JSON serialization use the same +struct. ## Files to modify -- `src/cmd/init.rs` — `format_text()`, result struct enrichment, `run()` takes `verbose` +- `src/cmd/init.rs` — `format_text()`, result struct enrichment, `run()` takes + `verbose` diff --git a/docs/spec/todos/TODO-0041.md b/docs/spec/todos/TODO-0041.md index 54e91cb..63d053a 100644 --- a/docs/spec/todos/TODO-0041.md +++ b/docs/spec/todos/TODO-0041.md @@ -12,7 +12,8 @@ blocks: [] ## Summary -Compact: change table (field/action/type + truncated globs). Verbose: record tables with glob details per field change, footer with glob + elapsed. +Compact: change table (field/action/type + truncated globs). Verbose: record +tables with glob details per field change, footer with glob + elapsed. ## Compact @@ -32,8 +33,13 @@ Record tables with "found in" / "previously in" details per field. ## Output alignment rule -Text and JSON must contain the same data — only the format differs. Verbose-only detail fields are `Option` with `#[serde(skip_serializing_if = "Option::is_none")]`, populated by `run(verbose: bool)`. Both `format_text()` and JSON serialization use the same struct. +Text and JSON must contain the same data — only the format differs. Verbose-only +detail fields are `Option` with +`#[serde(skip_serializing_if = "Option::is_none")]`, populated by +`run(verbose: bool)`. Both `format_text()` and JSON serialization use the same +struct. ## Files to modify -- `src/cmd/update.rs` — `format_text()`, result struct enrichment, `run()` takes `verbose` +- `src/cmd/update.rs` — `format_text()`, result struct enrichment, `run()` takes + `verbose` diff --git a/docs/spec/todos/TODO-0042.md b/docs/spec/todos/TODO-0042.md index 5d70fb0..33923bb 100644 --- a/docs/spec/todos/TODO-0042.md +++ b/docs/spec/todos/TODO-0042.md @@ -12,7 +12,10 @@ blocks: [] ## Summary -Most complex rewrite. Compact: metadata table (model/config/files) + field rules table (with truncated globs). Verbose: full metadata table + record tables per field with required/allowed, footer. Replace `status: up to date` with `config: match`/`changed` + `files: N/N` (indexed/on-disk). +Most complex rewrite. Compact: metadata table (model/config/files) + field rules +table (with truncated globs). Verbose: full metadata table + record tables per +field with required/allowed, footer. Replace `status: up to date` with +`config: match`/`changed` + `files: N/N` (indexed/on-disk). ## Compact @@ -34,18 +37,25 @@ Most complex rewrite. Compact: metadata table (model/config/files) + field rules ## Verbose -Full metadata table (model/revision/chunk size/built/config/files) + record tables per field with counts + required/allowed glob lists. +Full metadata table (model/revision/chunk size/built/config/files) + record +tables per field with counts + required/allowed glob lists. ## Changes -- Replace `config_match: bool` with `config: match`/`changed — rebuild recommended` +- Replace `config_match: bool` with + `config: match`/`changed — rebuild recommended` - Add `files: N/N` (indexed vs on-disk) - Verbose adds field counts (N/total) and full glob lists ## Output alignment rule -Text and JSON must contain the same data — only the format differs. Verbose-only detail fields are `Option` with `#[serde(skip_serializing_if = "Option::is_none")]`, populated by `run(verbose: bool)`. Both `format_text()` and JSON serialization use the same struct. +Text and JSON must contain the same data — only the format differs. Verbose-only +detail fields are `Option` with +`#[serde(skip_serializing_if = "Option::is_none")]`, populated by +`run(verbose: bool)`. Both `format_text()` and JSON serialization use the same +struct. ## Files to modify -- `src/cmd/info.rs` — `format_text()`, `IndexInfo` struct changes, `run()` takes `verbose` +- `src/cmd/info.rs` — `format_text()`, `IndexInfo` struct changes, `run()` takes + `verbose` diff --git a/docs/spec/todos/TODO-0043.md b/docs/spec/todos/TODO-0043.md index bf5e9bf..05492a7 100644 --- a/docs/spec/todos/TODO-0043.md +++ b/docs/spec/todos/TODO-0043.md @@ -12,21 +12,25 @@ blocks: [] ## Summary -Independent from output formatting. Currently debug and trace produce identical output, and neither shows elapsed times. Add distinct events at each level. +Independent from output formatting. Currently debug and trace produce identical +output, and neither shows elapsed times. Add distinct events at each level. ## Changes ### Debug level (`--logs=debug`) + - Per-file classify details (new/edited/unchanged/removed) in build - Per-file scan hits in discover - Per-file embed timing ### Trace level (`--logs=trace`) + - SQL queries in search/backend - Arrow schema details - Chunking decisions (chunk boundaries, sizes) ### Elapsed times + - Add `elapsed_ms` to debug/trace events where useful - Use `std::time::Instant` for per-operation timing diff --git a/docs/spec/todos/TODO-0044.md b/docs/spec/todos/TODO-0044.md index 4df680f..290706b 100644 --- a/docs/spec/todos/TODO-0044.md +++ b/docs/spec/todos/TODO-0044.md @@ -12,12 +12,16 @@ blocks: [45] ## Summary -Finalize `Cargo.toml` metadata for crates.io publication. Add `homepage`, `documentation`, and `repository` fields. Replace the broad `exclude` list with a surgical `include` list to guarantee a minimal source package. Verify with `cargo publish --dry-run`. +Finalize `Cargo.toml` metadata for crates.io publication. Add `homepage`, +`documentation`, and `repository` fields. Replace the broad `exclude` list with +a surgical `include` list to guarantee a minimal source package. Verify with +`cargo publish --dry-run`. ## Changes - Add `homepage`, `documentation`, `repository` to `[package]` -- Replace `exclude` with `include` list (src, LICENSE, README.md, Cargo.toml, Cargo.lock) +- Replace `exclude` with `include` list (src, LICENSE, README.md, Cargo.toml, + Cargo.lock) - Verify `.crate` file size under 1MB with `cargo publish --dry-run` ## Files to modify diff --git a/docs/spec/todos/TODO-0045.md b/docs/spec/todos/TODO-0045.md index ff33af7..2504ed2 100644 --- a/docs/spec/todos/TODO-0045.md +++ b/docs/spec/todos/TODO-0045.md @@ -12,7 +12,10 @@ blocks: [46, 47] ## Summary -Initialize `cargo-dist` to generate GitHub Actions release workflows. Configure build targets for cross-platform binary distribution: `aarch64-apple-darwin` (Mac M1+), `x86_64-apple-darwin` (Intel Mac), `x86_64-unknown-linux-gnu` (Linux), `x86_64-pc-windows-msvc` (Windows). +Initialize `cargo-dist` to generate GitHub Actions release workflows. Configure +build targets for cross-platform binary distribution: `aarch64-apple-darwin` +(Mac M1+), `x86_64-apple-darwin` (Intel Mac), `x86_64-unknown-linux-gnu` +(Linux), `x86_64-pc-windows-msvc` (Windows). ## Changes diff --git a/docs/spec/todos/TODO-0046.md b/docs/spec/todos/TODO-0046.md index 4395ae7..501a8bd 100644 --- a/docs/spec/todos/TODO-0046.md +++ b/docs/spec/todos/TODO-0046.md @@ -12,7 +12,8 @@ blocks: [] ## Summary -Enable Homebrew formula generation in cargo-dist config. On release, cargo-dist auto-updates a Homebrew tap repository with the new formula. Subsumes TODO-0030. +Enable Homebrew formula generation in cargo-dist config. On release, cargo-dist +auto-updates a Homebrew tap repository with the new formula. Subsumes TODO-0030. ## Changes diff --git a/docs/spec/todos/TODO-0047.md b/docs/spec/todos/TODO-0047.md index 010b793..3a3810e 100644 --- a/docs/spec/todos/TODO-0047.md +++ b/docs/spec/todos/TODO-0047.md @@ -12,7 +12,9 @@ blocks: [] ## Summary -Enable npm package generation in cargo-dist. The npm package acts as a binary wrapper — detects the platform and downloads the correct prebuilt binary from GitHub Releases. Enables `npx mdvs` usage. +Enable npm package generation in cargo-dist. The npm package acts as a binary +wrapper — detects the platform and downloads the correct prebuilt binary from +GitHub Releases. Enables `npx mdvs` usage. ## Changes diff --git a/docs/spec/todos/TODO-0048.md b/docs/spec/todos/TODO-0048.md index b9851da..de55cab 100644 --- a/docs/spec/todos/TODO-0048.md +++ b/docs/spec/todos/TODO-0048.md @@ -12,18 +12,26 @@ blocks: [] ## Summary -Update the README "Install" section to list the installation methods that are already available via cargo-dist. Drop Homebrew and npm as blockers — those can be added to the README when/if they land (TODO-0046, TODO-0047). +Update the README "Install" section to list the installation methods that are +already available via cargo-dist. Drop Homebrew and npm as blockers — those can +be added to the README when/if they land (TODO-0046, TODO-0047). ## Install methods to list (in order) -1. **Shell installer** (Mac + Linux) — `curl -sSL ... | sh` pointing at the GitHub release installer script -2. **PowerShell installer** (Windows) — `irm ... | iex` pointing at the GitHub release installer script -3. **Cargo** — `cargo install mdvs` (works once published to crates.io; repo will be open-sourced at the same time) +1. **Shell installer** (Mac + Linux) — `curl -sSL ... | sh` pointing at the + GitHub release installer script +2. **PowerShell installer** (Windows) — `irm ... | iex` pointing at the GitHub + release installer script +3. **Cargo** — `cargo install mdvs` (works once published to crates.io; repo + will be open-sourced at the same time) 4. **Build from source** — `git clone` + `cargo install --path .` ## Current state -The README only shows `cargo install mdvs` and build from source. The shell/PowerShell installers from cargo-dist (`mdvs-installer.sh`, `mdvs-installer.ps1`) are already in the GitHub release (`v0.0.2-rc.1`) but not documented. +The README only shows `cargo install mdvs` and build from source. The +shell/PowerShell installers from cargo-dist (`mdvs-installer.sh`, +`mdvs-installer.ps1`) are already in the GitHub release (`v0.0.2-rc.1`) but not +documented. ## Files to modify diff --git a/docs/spec/todos/TODO-0049.md b/docs/spec/todos/TODO-0049.md index fe43d71..06e219f 100644 --- a/docs/spec/todos/TODO-0049.md +++ b/docs/spec/todos/TODO-0049.md @@ -12,7 +12,8 @@ blocks: [] ## Summary -CI workflow at `.github/workflows/ci.yml` — runs on every push to `main` and every PR. +CI workflow at `.github/workflows/ci.yml` — runs on every push to `main` and +every PR. ## Resolution diff --git a/docs/spec/todos/TODO-0050.md b/docs/spec/todos/TODO-0050.md index f21eaa0..7a4c37b 100644 --- a/docs/spec/todos/TODO-0050.md +++ b/docs/spec/todos/TODO-0050.md @@ -12,7 +12,10 @@ blocks: [] ## Details -In `storage.rs:174-182`, when a YAML frontmatter field contains `null` and the field is typed as String, `Value::Null.to_string()` produces the literal string `"null"` instead of Arrow NULL. Other types (Boolean, Integer, Float) correctly produce NULL for null values. +In `storage.rs:174-182`, when a YAML frontmatter field contains `null` and the +field is typed as String, `Value::Null.to_string()` produces the literal string +`"null"` instead of Arrow NULL. Other types (Boolean, Integer, Float) correctly +produce NULL for null values. Fix: check for `Value::Null` before the match and return `None` (Arrow NULL). diff --git a/docs/spec/todos/TODO-0051.md b/docs/spec/todos/TODO-0051.md index d3e84c3..8a980ed 100644 --- a/docs/spec/todos/TODO-0051.md +++ b/docs/spec/todos/TODO-0051.md @@ -12,7 +12,10 @@ blocks: [] ## Details -In `embed.rs:46`, `Embedder::load()` uses `unwrap_or_else(|e| panic!(...))` on model loading failure. This crashes the process with a stacktrace instead of returning a helpful error. Should return `anyhow::Result` and propagate the error. +In `embed.rs:46`, `Embedder::load()` uses `unwrap_or_else(|e| panic!(...))` on +model loading failure. This crashes the process with a stacktrace instead of +returning a helpful error. Should return `anyhow::Result` and propagate the +error. ## Files diff --git a/docs/spec/todos/TODO-0052.md b/docs/spec/todos/TODO-0052.md index 0f89195..e3c3456 100644 --- a/docs/spec/todos/TODO-0052.md +++ b/docs/spec/todos/TODO-0052.md @@ -12,7 +12,9 @@ blocks: [] ## Details -In `scan.rs:60`, `.expect("failed to read file")` panics if a single file is unreadable (permission denied, encoding error, deleted between walk and read). Should skip the file with a warning instead of crashing the entire scan. +In `scan.rs:60`, `.expect("failed to read file")` panics if a single file is +unreadable (permission denied, encoding error, deleted between walk and read). +Should skip the file with a warning instead of crashing the entire scan. ## Files diff --git a/docs/spec/todos/TODO-0053.md b/docs/spec/todos/TODO-0053.md index 1798718..557adfc 100644 --- a/docs/spec/todos/TODO-0053.md +++ b/docs/spec/todos/TODO-0053.md @@ -12,7 +12,9 @@ blocks: [] ## Details -In `scan.rs:54`, `.unwrap()` on `strip_prefix(root)` will panic if a symlink causes a file's absolute path to be outside the root directory. Should skip the file with a warning or return an error. +In `scan.rs:54`, `.unwrap()` on `strip_prefix(root)` will panic if a symlink +causes a file's absolute path to be outside the root directory. Should skip the +file with a warning or return an error. ## Files diff --git a/docs/spec/todos/TODO-0054.md b/docs/spec/todos/TODO-0054.md index 9584202..3519855 100644 --- a/docs/spec/todos/TODO-0054.md +++ b/docs/spec/todos/TODO-0054.md @@ -12,7 +12,9 @@ blocks: [] ## Details -In `scan.rs:34`, `.expect("invalid glob pattern")` panics on an invalid user-provided glob pattern from `mdvs.toml`. Should return an `anyhow::Result` with a message showing the invalid pattern. +In `scan.rs:34`, `.expect("invalid glob pattern")` panics on an invalid +user-provided glob pattern from `mdvs.toml`. Should return an `anyhow::Result` +with a message showing the invalid pattern. ## Files diff --git a/docs/spec/todos/TODO-0055.md b/docs/spec/todos/TODO-0055.md index 765cf7b..be1583e 100644 --- a/docs/spec/todos/TODO-0055.md +++ b/docs/spec/todos/TODO-0055.md @@ -12,7 +12,9 @@ blocks: [] ## Details -`scan.rs` reads files entirely into memory with `fs::read_to_string()`. A very large file (e.g. 10GB) causes OOM. Add a configurable file size limit (e.g. 100MB default) and skip files that exceed it with a warning. +`scan.rs` reads files entirely into memory with `fs::read_to_string()`. A very +large file (e.g. 10GB) causes OOM. Add a configurable file size limit (e.g. +100MB default) and skip files that exceed it with a warning. ## Files diff --git a/docs/spec/todos/TODO-0056.md b/docs/spec/todos/TODO-0056.md index 13629e1..9578720 100644 --- a/docs/spec/todos/TODO-0056.md +++ b/docs/spec/todos/TODO-0056.md @@ -12,7 +12,11 @@ blocks: [] ## Details -`clean.rs` calls `remove_dir_all` on `.mdvs/` without checking if it's a symlink. An attacker could replace `.mdvs/` with a symlink to another directory, causing `clean` to delete unintended files. Should verify `.mdvs/` is a real directory and optionally check it contains expected files (e.g. `files.parquet`). +`clean.rs` calls `remove_dir_all` on `.mdvs/` without checking if it's a +symlink. An attacker could replace `.mdvs/` with a symlink to another directory, +causing `clean` to delete unintended files. Should verify `.mdvs/` is a real +directory and optionally check it contains expected files (e.g. +`files.parquet`). ## Files diff --git a/docs/spec/todos/TODO-0057.md b/docs/spec/todos/TODO-0057.md index 89b1e1a..f143421 100644 --- a/docs/spec/todos/TODO-0057.md +++ b/docs/spec/todos/TODO-0057.md @@ -12,7 +12,11 @@ blocks: [] ## Details -`build::run()` is ~312 lines and handles too many responsibilities. The model loading + dimension check logic is duplicated between the full rebuild path (~line 459) and the incremental rebuild path (~line 562). Extract into a shared helper function. Consider also extracting config filling and config change detection into separate functions. +`build::run()` is ~312 lines and handles too many responsibilities. The model +loading + dimension check logic is duplicated between the full rebuild path +(~line 459) and the incremental rebuild path (~line 562). Extract into a shared +helper function. Consider also extracting config filling and config change +detection into separate functions. ## Files diff --git a/docs/spec/todos/TODO-0058.md b/docs/spec/todos/TODO-0058.md index e6cff51..2eda359 100644 --- a/docs/spec/todos/TODO-0058.md +++ b/docs/spec/todos/TODO-0058.md @@ -12,7 +12,8 @@ blocks: [] ## Details -In `search.rs:147,149`, `.to_str().unwrap()` on parquet file paths will panic on non-UTF-8 paths. Should use `.ok_or_else()` with a descriptive error message. +In `search.rs:147,149`, `.to_str().unwrap()` on parquet file paths will panic on +non-UTF-8 paths. Should use `.ok_or_else()` with a descriptive error message. ## Files diff --git a/docs/spec/todos/TODO-0059.md b/docs/spec/todos/TODO-0059.md index cc89efa..6f633db 100644 --- a/docs/spec/todos/TODO-0059.md +++ b/docs/spec/todos/TODO-0059.md @@ -12,7 +12,9 @@ blocks: [] ## Details -In `output.rs:141`, `serde_json::to_string_pretty(self).unwrap()` panics if JSON serialization fails. This is the default `print()` implementation used by all commands. Should use `.expect()` with a message or return a Result. +In `output.rs:141`, `serde_json::to_string_pretty(self).unwrap()` panics if JSON +serialization fails. This is the default `print()` implementation used by all +commands. Should use `.expect()` with a message or return a Result. ## Files diff --git a/docs/spec/todos/TODO-0060.md b/docs/spec/todos/TODO-0060.md index aaaedf0..ac9ff71 100644 --- a/docs/spec/todos/TODO-0060.md +++ b/docs/spec/todos/TODO-0060.md @@ -12,7 +12,8 @@ blocks: [] ## Details -`cmd/update.rs` has zero tests. This is a critical command (re-inference, field management). Needs tests for: +`cmd/update.rs` has zero tests. This is a critical command (re-inference, field +management). Needs tests for: - New fields only (default mode) - `--reinfer ` re-infers named field, keeps others diff --git a/docs/spec/todos/TODO-0061.md b/docs/spec/todos/TODO-0061.md index e5c14e6..0ae18d9 100644 --- a/docs/spec/todos/TODO-0061.md +++ b/docs/spec/todos/TODO-0061.md @@ -12,7 +12,9 @@ blocks: [] ## Details -No test verifies that `build` aborts before embedding when `check::validate()` finds violations. This is a critical safety property — dirty data should never enter the parquets. Also add tests for: +No test verifies that `build` aborts before embedding when `check::validate()` +finds violations. This is a critical safety property — dirty data should never +enter the parquets. Also add tests for: - Build with model mismatch (requires `--force`) - Build with manual toml edits (config change detection) diff --git a/docs/spec/todos/TODO-0062.md b/docs/spec/todos/TODO-0062.md index c1466bc..a18727a 100644 --- a/docs/spec/todos/TODO-0062.md +++ b/docs/spec/todos/TODO-0062.md @@ -12,7 +12,10 @@ blocks: [] ## Details -No test verifies that `Value::Null` in String-typed fields survives a parquet write→read roundtrip as Arrow NULL (not the string `"null"`). This covers the bug in TODO-0050. Also add tests for null values in Boolean, Integer, and Float fields to prevent regression. +No test verifies that `Value::Null` in String-typed fields survives a parquet +write→read roundtrip as Arrow NULL (not the string `"null"`). This covers the +bug in TODO-0050. Also add tests for null values in Boolean, Integer, and Float +fields to prevent regression. ## Files diff --git a/docs/spec/todos/TODO-0066.md b/docs/spec/todos/TODO-0066.md index d703112..6518873 100644 --- a/docs/spec/todos/TODO-0066.md +++ b/docs/spec/todos/TODO-0066.md @@ -12,7 +12,9 @@ blocks: [] ## Details -Deeply nested YAML frontmatter (1000+ levels) can exhaust stack memory during parsing, causing a crash. Add a depth limit or validation step after parsing to reject excessively nested structures. +Deeply nested YAML frontmatter (1000+ levels) can exhaust stack memory during +parsing, causing a crash. Add a depth limit or validation step after parsing to +reject excessively nested structures. ## Files diff --git a/docs/spec/todos/TODO-0067.md b/docs/spec/todos/TODO-0067.md index d7b394d..4dbb587 100644 --- a/docs/spec/todos/TODO-0067.md +++ b/docs/spec/todos/TODO-0067.md @@ -12,7 +12,9 @@ blocks: [] ## Details -No limit on the number of frontmatter keys per file. A file with 100,000 keys would consume excessive memory and slow inference/validation. Add a configurable limit (e.g. 1000 fields max) with a warning or error. +No limit on the number of frontmatter keys per file. A file with 100,000 keys +would consume excessive memory and slow inference/validation. Add a configurable +limit (e.g. 1000 fields max) with a warning or error. ## Files diff --git a/docs/spec/todos/TODO-0068.md b/docs/spec/todos/TODO-0068.md index ec446ed..53e29e3 100644 --- a/docs/spec/todos/TODO-0068.md +++ b/docs/spec/todos/TODO-0068.md @@ -12,7 +12,10 @@ blocks: [] ## Details -Parquet deserialization errors in `storage.rs` use generic messages like `"expected StringArray for file_id"` without indicating which parquet file, batch, or row caused the error. Add `.context()` calls with file path and column information. +Parquet deserialization errors in `storage.rs` use generic messages like +`"expected StringArray for file_id"` without indicating which parquet file, +batch, or row caused the error. Add `.context()` calls with file path and column +information. ## Files diff --git a/docs/spec/todos/TODO-0069.md b/docs/spec/todos/TODO-0069.md index de35aa6..284e67f 100644 --- a/docs/spec/todos/TODO-0069.md +++ b/docs/spec/todos/TODO-0069.md @@ -12,7 +12,10 @@ blocks: [] ## Details -In `search.rs:101-109`, `read_lines()` silently returns `None` if a file is unreadable (deleted, moved, or permission denied since build). In verbose mode, the user sees an empty `chunk_text` with no explanation. Should log a warning via `tracing::warn!`. +In `search.rs:101-109`, `read_lines()` silently returns `None` if a file is +unreadable (deleted, moved, or permission denied since build). In verbose mode, +the user sees an empty `chunk_text` with no explanation. Should log a warning +via `tracing::warn!`. ## Files diff --git a/docs/spec/todos/TODO-0070.md b/docs/spec/todos/TODO-0070.md index ea3586e..8d979e3 100644 --- a/docs/spec/todos/TODO-0070.md +++ b/docs/spec/todos/TODO-0070.md @@ -12,7 +12,9 @@ blocks: [] ## Details -`init::run()` and `update::run()` both scan files, run inference, and build/update `MdvsToml`. The inference + config update flow could be extracted into a shared utility function to reduce duplication and ensure consistency. +`init::run()` and `update::run()` both scan files, run inference, and +build/update `MdvsToml`. The inference + config update flow could be extracted +into a shared utility function to reduce duplication and ensure consistency. ## Files diff --git a/docs/spec/todos/TODO-0071.md b/docs/spec/todos/TODO-0071.md index 8fee32a..d772390 100644 --- a/docs/spec/todos/TODO-0071.md +++ b/docs/spec/todos/TODO-0071.md @@ -12,7 +12,10 @@ blocks: [] ## Details -`check::validate()` is ~150 lines handling all validation phases (missing required, wrong type, disallowed fields, new fields detection). Breaking it into smaller functions (`check_required`, `check_types`, `check_disallowed`) would improve testability — each sub-check could be unit-tested independently. +`check::validate()` is ~150 lines handling all validation phases (missing +required, wrong type, disallowed fields, new fields detection). Breaking it into +smaller functions (`check_required`, `check_types`, `check_disallowed`) would +improve testability — each sub-check could be unit-tested independently. ## Files diff --git a/docs/spec/todos/TODO-0072.md b/docs/spec/todos/TODO-0072.md index 0bbdd3c..7ef1f3c 100644 --- a/docs/spec/todos/TODO-0072.md +++ b/docs/spec/todos/TODO-0072.md @@ -22,16 +22,28 @@ files_updated: ## Summary -Special characters in field names and values can break the SQL strings constructed for DataFusion queries. Field names with quotes break the `CREATE VIEW` SQL in `search.rs`. Values with quotes (e.g. `O'Brien`) break user-supplied `--where` clauses. +Special characters in field names and values can break the SQL strings +constructed for DataFusion queries. Field names with quotes break the +`CREATE VIEW` SQL in `search.rs`. Values with quotes (e.g. `O'Brien`) break +user-supplied `--where` clauses. ## Resolution Implemented three layers of protection: -1. **VIEW SQL escaping** (`search.rs`): Field names interpolated into `CREATE VIEW` now escape single quotes in dictionary accessors (`'` → `''`) and double quotes in column aliases (`"` → `""`). +1. **VIEW SQL escaping** (`search.rs`): Field names interpolated into + `CREATE VIEW` now escape single quotes in dictionary accessors (`'` → `''`) + and double quotes in column aliases (`"` → `""`). -2. **`--where` parity check** (`cmd/search.rs`): Before passing the clause to DataFusion, validate that single and double quotes are balanced (odd count = unescaped). Provides clear error messages with escaping examples. +2. **`--where` parity check** (`cmd/search.rs`): Before passing the clause to + DataFusion, validate that single and double quotes are balanced (odd count = + unescaped). Provides clear error messages with escaping examples. -3. **FieldHint system** (`output.rs`): `FieldHint` enum with `EscapeSingleQuotes` and `EscapeDoubleQuotes` variants. `field_hints()` computes hints from field names. Hints shown in init/update/info command output (both text and JSON) to warn users about fields requiring SQL escaping in `--where` clauses. +3. **FieldHint system** (`output.rs`): `FieldHint` enum with + `EscapeSingleQuotes` and `EscapeDoubleQuotes` variants. `field_hints()` + computes hints from field names. Hints shown in init/update/info command + output (both text and JSON) to warn users about fields requiring SQL escaping + in `--where` clauses. -4. **`--where` help text** (`main.rs`): Added `long_help` with escaping examples to the `--where` clap argument. +4. **`--where` help text** (`main.rs`): Added `long_help` with escaping examples + to the `--where` clap argument. diff --git a/docs/spec/todos/TODO-0073.md b/docs/spec/todos/TODO-0073.md index 5f371e3..83a87e2 100644 --- a/docs/spec/todos/TODO-0073.md +++ b/docs/spec/todos/TODO-0073.md @@ -18,12 +18,21 @@ files_updated: ## Summary -When `build` aborts due to validation violations, the formatted check result (tables) is embedded in `anyhow::bail!()`, sending it to stderr as unstructured error text. For `--output json`, no JSON is produced at all. +When `build` aborts due to validation violations, the formatted check result +(tables) is embedded in `anyhow::bail!()`, sending it to stderr as unstructured +error text. For `--output json`, no JSON is produced at all. ## Resolution -Added `BuildOutcome` enum to `build.rs` with `Success(BuildResult)` and `ValidationFailed(CheckResult)` variants. `build::run()` returns `Ok(BuildOutcome::ValidationFailed(check_result))` instead of bailing. `main.rs` matches on the outcome — prints `CheckResult` to stdout via `CommandOutput::print()`, then calls `process::exit(1)`. Same pattern as `check`. +Added `BuildOutcome` enum to `build.rs` with `Success(BuildResult)` and +`ValidationFailed(CheckResult)` variants. `build::run()` returns +`Ok(BuildOutcome::ValidationFailed(check_result))` instead of bailing. `main.rs` +matches on the outcome — prints `CheckResult` to stdout via +`CommandOutput::print()`, then calls `process::exit(1)`. Same pattern as +`check`. -`init.rs` and `update.rs` also updated to handle `BuildOutcome` — validation failure after inference is treated as a bug (bails with descriptive message). +`init.rs` and `update.rs` also updated to handle `BuildOutcome` — validation +failure after inference is treated as a bug (bails with descriptive message). -Removed `depends_on: [78]` — this fix is independent. TODO-0078 tracks the broader step-based output model. +Removed `depends_on: [78]` — this fix is independent. TODO-0078 tracks the +broader step-based output model. diff --git a/docs/spec/todos/TODO-0074.md b/docs/spec/todos/TODO-0074.md index 314ec00..e1fe36a 100644 --- a/docs/spec/todos/TODO-0074.md +++ b/docs/spec/todos/TODO-0074.md @@ -12,17 +12,25 @@ blocks: [] ## Summary -`content_hash()` uses `std::hash::DefaultHasher` (SipHash), whose output is not guaranteed stable across Rust compiler versions. Rebuilding mdvs with a new toolchain could invalidate all stored hashes, causing unnecessary full re-embedding on the next incremental build. +`content_hash()` uses `std::hash::DefaultHasher` (SipHash), whose output is not +guaranteed stable across Rust compiler versions. Rebuilding mdvs with a new +toolchain could invalidate all stored hashes, causing unnecessary full +re-embedding on the next incremental build. ## Decision -Use `xxhash-rust` with the `xxh3` feature (`xxh3_64`). Fast, stable output across platforms, widely used for content hashing. No cryptographic properties needed here. +Use `xxhash-rust` with the `xxh3` feature (`xxh3_64`). Fast, stable output +across platforms, widely used for content hashing. No cryptographic properties +needed here. ## Details -The hash is stored as a 16-char hex `String` in parquet (`content_hash` column, `Utf8`). Changing the algorithm requires a `--force` rebuild (existing hashes won't match), so this is a one-time migration cost. +The hash is stored as a 16-char hex `String` in parquet (`content_hash` column, +`Utf8`). Changing the algorithm requires a `--force` rebuild (existing hashes +won't match), so this is a one-time migration cost. -Note: this is low priority because the practical impact is just an unexpected full rebuild after a Rust toolchain upgrade, not data corruption. +Note: this is low priority because the practical impact is just an unexpected +full rebuild after a Rust toolchain upgrade, not data corruption. ## Implementation @@ -33,6 +41,7 @@ In `Cargo.toml`, add `xxhash-rust` with the `xxh3` feature flag. ### 2. Replace `content_hash()` in `src/index/storage.rs` **Current** (line 50-55): + ```rust use std::hash::{DefaultHasher, Hasher}; @@ -44,6 +53,7 @@ pub fn content_hash(content: &str) -> String { ``` **New:** + ```rust use xxhash_rust::xxh3::xxh3_64; @@ -52,26 +62,35 @@ pub fn content_hash(content: &str) -> String { } ``` -Remove the `use std::hash::{DefaultHasher, Hasher};` import (line 5) — nothing else uses it. +Remove the `use std::hash::{DefaultHasher, Hasher};` import (line 5) — nothing +else uses it. ### 3. Update doc comment -Change `FileIndexEntry.content_hash` doc comment (line 433) from "SipHash of the markdown body" to "xxh3 hash of the markdown body". +Change `FileIndexEntry.content_hash` doc comment (line 433) from "SipHash of the +markdown body" to "xxh3 hash of the markdown body". ### 4. No other changes needed - Function signature unchanged: `pub fn content_hash(content: &str) -> String` - Output format unchanged: 16-char lowercase hex (`u64`) -- All callers unchanged — 3 call sites in `src/cmd/build.rs` (lines 262, 529, 689) import and call identically -- Test `content_hash_determinism` (line 1259) still passes — it only checks same→same, different→different -- Tests with hardcoded hash strings (`"h1"`, `"hash1"`, etc.) never call `content_hash()`, unaffected +- All callers unchanged — 3 call sites in `src/cmd/build.rs` (lines 262, + 529, 689) import and call identically +- Test `content_hash_determinism` (line 1259) still passes — it only checks + same→same, different→different +- Tests with hardcoded hash strings (`"h1"`, `"hash1"`, etc.) never call + `content_hash()`, unaffected - `FileIndexEntry` struct, `read_file_index()`, parquet schema — all unchanged ## Migration -Existing parquets contain SipHash values. After this change, the first incremental build will see every file as "edited" (hash mismatch) and re-embed all files. This is the expected one-time cost. No `--force` required — it's the same behavior as editing every file. +Existing parquets contain SipHash values. After this change, the first +incremental build will see every file as "edited" (hash mismatch) and re-embed +all files. This is the expected one-time cost. No `--force` required — it's the +same behavior as editing every file. ## Files - `Cargo.toml` — add `xxhash-rust = { version = "...", features = ["xxh3"] }` -- `src/index/storage.rs` — `content_hash()` function (line 51), remove `DefaultHasher` import (line 5), update `FileIndexEntry` doc (line 433) +- `src/index/storage.rs` — `content_hash()` function (line 51), remove + `DefaultHasher` import (line 5), update `FileIndexEntry` doc (line 433) diff --git a/docs/spec/todos/TODO-0075.md b/docs/spec/todos/TODO-0075.md index c3bb8ad..2735183 100644 --- a/docs/spec/todos/TODO-0075.md +++ b/docs/spec/todos/TODO-0075.md @@ -12,51 +12,63 @@ blocks: [] ## Summary -Array fields like `tags: [rust, traits]` can't be filtered with simple equality in `--where`. Users need DataFusion-specific array functions. This TODO tracks what works, what doesn't, and what (if anything) we should do about it. +Array fields like `tags: [rust, traits]` can't be filtered with simple equality +in `--where`. Users need DataFusion-specific array functions. This TODO tracks +what works, what doesn't, and what (if anything) we should do about it. ## Findings -Tested empirically against DataFusion 52 with `List` columns promoted through the `files_v` view (same mechanism as production). Test script: `scripts/test_array_queries.rs`. +Tested empirically against DataFusion 52 with `List` columns promoted +through the `files_v` view (same mechanism as production). Test script: +`scripts/test_array_queries.rs`. ### Array queries — all work out of the box -| Syntax | Example | Works? | -|--------|---------|--------| -| `array_has(col, val)` | `array_has(tags, 'rust')` | Yes — via both bracket accessor (`data['tags']`) and view alias (`tags`) | -| `array_contains(col, val)` | `array_contains(tags, 'rust')` | Yes — alias for `array_has` | -| `val = ANY(col)` | `'rust' = ANY(tags)` | Yes — SQL standard syntax | -| `array_length(col)` | `array_length(tags) > 2` | Yes — returns `UInt64` | -| `UNNEST(col)` | `UNNEST(tags) AS tag` | Yes — row expansion, works with `GROUP BY` | -| Multiple `array_has` with `AND` | `array_has(tags, 'rust') AND array_has(tags, 'async')` | Yes | -| `array_has` + scalar filter | `array_has(tags, 'rust') AND draft = false` | Yes | +| Syntax | Example | Works? | +| ------------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------ | +| `array_has(col, val)` | `array_has(tags, 'rust')` | Yes — via both bracket accessor (`data['tags']`) and view alias (`tags`) | +| `array_contains(col, val)` | `array_contains(tags, 'rust')` | Yes — alias for `array_has` | +| `val = ANY(col)` | `'rust' = ANY(tags)` | Yes — SQL standard syntax | +| `array_length(col)` | `array_length(tags) > 2` | Yes — returns `UInt64` | +| `UNNEST(col)` | `UNNEST(tags) AS tag` | Yes — row expansion, works with `GROUP BY` | +| Multiple `array_has` with `AND` | `array_has(tags, 'rust') AND array_has(tags, 'async')` | Yes | +| `array_has` + scalar filter | `array_has(tags, 'rust') AND draft = false` | Yes | ### Nested Struct queries — also work -The view promotes `_data` children one level deep. If a child is itself a Struct (e.g. `meta: { source: "web", reviewed: true }`), it's promoted as a Struct column. Bracket access works on the promoted column: +The view promotes `_data` children one level deep. If a child is itself a Struct +(e.g. `meta: { source: "web", reviewed: true }`), it's promoted as a Struct +column. Bracket access works on the promoted column: -| Syntax | Example | Works? | -|--------|---------|--------| -| `meta['field']` via view | `meta['source'] = 'web'` | Yes | -| `data['meta']['field']` via raw table | `data['meta']['source'] = 'web'` | Yes — chained bracket access | -| `meta['bool_field']` | `meta['reviewed'] = true` | Yes | -| Combined array + nested struct | `array_has(tags, 'rust') AND meta['source'] = 'web'` | Yes | +| Syntax | Example | Works? | +| ------------------------------------- | ---------------------------------------------------- | ---------------------------- | +| `meta['field']` via view | `meta['source'] = 'web'` | Yes | +| `data['meta']['field']` via raw table | `data['meta']['source'] = 'web'` | Yes — chained bracket access | +| `meta['bool_field']` | `meta['reviewed'] = true` | Yes | +| Combined array + nested struct | `array_has(tags, 'rust') AND meta['source'] = 'web'` | Yes | ### Edge cases - `array_has(tags, 'nonexistent')` returns 0 rows (no error) -- `UNNEST` + `GROUP BY` works for tag frequency queries: `SELECT UNNEST(tags) AS tag, COUNT(*) AS cnt FROM files_v GROUP BY tag ORDER BY cnt DESC` +- `UNNEST` + `GROUP BY` works for tag frequency queries: + `SELECT UNNEST(tags) AS tag, COUNT(*) AS cnt FROM files_v GROUP BY tag ORDER BY cnt DESC` ## Decision -No code changes needed — DataFusion 52 natively supports all useful array and nested struct query patterns through the `files_v` view. +No code changes needed — DataFusion 52 natively supports all useful array and +nested struct query patterns through the `files_v` view. -Spec updated: `docs/spec/commands/search.md` now documents all query patterns (scalar, array, nested object, special characters, legacy bracket syntax) with `example_kb/` examples. +Spec updated: `docs/spec/commands/search.md` now documents all query patterns +(scalar, array, nested object, special characters, legacy bracket syntax) with +`example_kb/` examples. User-facing docs (book) covered by TODO-0029. ## Remaining work -1. **Add test coverage** in `src/search.rs` — the existing test data only has `title: String` and `draft: Boolean`. Add a test with array fields and verify `array_has` works through the view. +1. **Add test coverage** in `src/search.rs` — the existing test data only has + `title: String` and `draft: Boolean`. Add a test with array fields and verify + `array_has` works through the view. ## Files diff --git a/docs/spec/todos/TODO-0076.md b/docs/spec/todos/TODO-0076.md index b5e987a..dc90c99 100644 --- a/docs/spec/todos/TODO-0076.md +++ b/docs/spec/todos/TODO-0076.md @@ -12,13 +12,19 @@ blocks: [] ## Summary -Field names with spaces (common in Obsidian vaults) require double-quoted SQL identifiers in `--where` clauses, which clashes with shell quoting and creates a bad UX: `--where "\"my field\" = 'value'"`. +Field names with spaces (common in Obsidian vaults) require double-quoted SQL +identifiers in `--where` clauses, which clashes with shell quoting and creates a +bad UX: `--where "\"my field\" = 'value'"`. ## Decision -Add a `ContainsSpaces` hint to the existing `FieldHint` enum. This surfaces the problem in `info` output so users know which fields need quoting. No code-level fix for the quoting pain itself — that would require user-defined aliases in `mdvs.toml`, which is a heavier feature for a future TODO. +Add a `ContainsSpaces` hint to the existing `FieldHint` enum. This surfaces the +problem in `info` output so users know which fields need quoting. No code-level +fix for the quoting pain itself — that would require user-defined aliases in +`mdvs.toml`, which is a heavier feature for a future TODO. -Auto-aliasing (`my field` → `my_field` in the view) was considered but rejected — risk of collision with existing fields that already use underscores. +Auto-aliasing (`my field` → `my_field` in the view) was considered but rejected +— risk of collision with existing fields that already use underscores. ## Implementation @@ -31,6 +37,7 @@ ContainsSpaces, ``` Add `Display` arm: + ```rust FieldHint::ContainsSpaces => write!(f, "use \"field name\" in --where"), ``` @@ -38,6 +45,7 @@ FieldHint::ContainsSpaces => write!(f, "use \"field name\" in --where"), ### 2. Update `field_hints()` (`src/output.rs`) Add space detection alongside the existing quote checks: + ```rust if name.contains(' ') { hints.push(FieldHint::ContainsSpaces); @@ -47,18 +55,24 @@ if name.contains(' ') { ### 3. Add tests (`src/output.rs`) - `field_hints_spaces()` — `"my field"` → `[ContainsSpaces]` -- `field_hints_spaces_and_quotes()` — `"author's field"` → `[EscapeSingleQuotes, ContainsSpaces]` +- `field_hints_spaces_and_quotes()` — `"author's field"` → + `[EscapeSingleQuotes, ContainsSpaces]` ### 4. Update docs -Add a note in `docs/book/src/searching.md` about quoting field names with spaces: +Add a note in `docs/book/src/searching.md` about quoting field names with +spaces: + ```bash mdvs search "query" --where '"my field" = '\''value'\''' ``` ## Future direction -If space-containing fields turn out to be common enough that the quoting UX is a real pain point, consider adding `alias = "my_field"` on `[[fields.field]]` in `mdvs.toml`. The alias would be used in the view instead of the raw field name, giving users a clean identifier to use in `--where`. +If space-containing fields turn out to be common enough that the quoting UX is a +real pain point, consider adding `alias = "my_field"` on `[[fields.field]]` in +`mdvs.toml`. The alias would be used in the view instead of the raw field name, +giving users a clean identifier to use in `--where`. ## Files diff --git a/docs/spec/todos/TODO-0077.md b/docs/spec/todos/TODO-0077.md index d4bedba..88e41e6 100644 --- a/docs/spec/todos/TODO-0077.md +++ b/docs/spec/todos/TODO-0077.md @@ -12,7 +12,9 @@ blocks: [] ## Summary -Add a repeatable `--field ` flag to `mdvs info` so users can see info for specific fields instead of all fields. Exact match only (no globs). Error if a requested field name doesn't exist in the toml. +Add a repeatable `--field ` flag to `mdvs info` so users can see info for +specific fields instead of all fields. Exact match only (no globs). Error if a +requested field name doesn't exist in the toml. ## Details @@ -35,24 +37,33 @@ Pass `fields` into `info::run()`. ### Filtering in `info::run()` (`src/cmd/info.rs`) -After building the `Vec` from toml (line 237-255), filter it down to only the requested names. If `--field` is not provided (empty vec), show all fields as today. +After building the `Vec` from toml (line 237-255), filter it down to +only the requested names. If `--field` is not provided (empty vec), show all +fields as today. -For each requested name, check it exists in `config.fields.field` (by name). If any requested name is not found, return an error like: +For each requested name, check it exists in `config.fields.field` (by name). If +any requested name is not found, return an error like: ``` Error: field "nonexistent" not found in mdvs.toml ``` -Check all names before returning, so the user sees all missing names at once (not one at a time). +Check all names before returning, so the user sees all missing names at once +(not one at a time). ### Output changes -- The one-liner summary should reflect the filtered count (e.g. "2 of 10 fields" or just show the filtered count). -- `ignored_fields` in `InfoResult` should be unaffected — it's a separate section. +- The one-liner summary should reflect the filtered count (e.g. "2 of 10 fields" + or just show the filtered count). +- `ignored_fields` in `InfoResult` should be unaffected — it's a separate + section. - Index info, scan glob, footer — all unaffected. -- JSON output: `fields` array contains only the filtered fields. No structural change to `InfoResult`. +- JSON output: `fields` array contains only the filtered fields. No structural + change to `InfoResult`. ### Files -- `src/main.rs` — add `fields: Vec` to `Info` variant, pass to `info::run()` -- `src/cmd/info.rs` — add `field_filter: &[String]` param to `run()`, filter fields, validate names exist +- `src/main.rs` — add `fields: Vec` to `Info` variant, pass to + `info::run()` +- `src/cmd/info.rs` — add `field_filter: &[String]` param to `run()`, filter + fields, validate names exist diff --git a/docs/spec/todos/TODO-0078.md b/docs/spec/todos/TODO-0078.md index 03b27a6..77e6e69 100644 --- a/docs/spec/todos/TODO-0078.md +++ b/docs/spec/todos/TODO-0078.md @@ -35,8 +35,19 @@ files_updated: ## Summary -Rework command output to follow a uniform pipeline model. Every command is a sequence of processing steps, each producing a typed result. The command output separates the processing record (what steps ran) from the final result. Errors are contextualized by the step where they occurred. Both text and JSON formats render consistently. +Rework command output to follow a uniform pipeline model. Every command is a +sequence of processing steps, each producing a typed result. The command output +separates the processing record (what steps ran) from the final result. Errors +are contextualized by the step where they occurred. Both text and JSON formats +render consistently. ## Resolution -Fully implemented across sub-TODOs 0079-0087. Core abstractions (`ProcessingStep`, `ProcessingStepResult`, `StepOutput`, `ErrorKind`) in `src/pipeline/mod.rs`. 12 step modules in `src/pipeline/`. All 7 commands reworked to flat pipelines returning `*CommandOutput` directly (no `anyhow::Result`). JSON output shows all steps with status/elapsed/output. Text output shows result on success, first failed step on failure. Exit code 2 for failed steps, exit code 1 for violations (check/build). Exit code 2/3 split (user vs application errors) deferred to TODO-0088 (paused). +Fully implemented across sub-TODOs 0079-0087. Core abstractions +(`ProcessingStep`, `ProcessingStepResult`, `StepOutput`, `ErrorKind`) in +`src/pipeline/mod.rs`. 12 step modules in `src/pipeline/`. All 7 commands +reworked to flat pipelines returning `*CommandOutput` directly (no +`anyhow::Result`). JSON output shows all steps with status/elapsed/output. Text +output shows result on success, first failed step on failure. Exit code 2 for +failed steps, exit code 1 for violations (check/build). Exit code 2/3 split +(user vs application errors) deferred to TODO-0088 (paused). diff --git a/docs/spec/todos/TODO-0079.md b/docs/spec/todos/TODO-0079.md index a5333d7..f3c3149 100644 --- a/docs/spec/todos/TODO-0079.md +++ b/docs/spec/todos/TODO-0079.md @@ -15,8 +15,14 @@ files_created: ## Summary -Define the foundational types for the step-based pipeline model in a new `src/pipeline/mod.rs` module. +Define the foundational types for the step-based pipeline model in a new +`src/pipeline/mod.rs` module. ## Resolution -Implemented `StepOutput` trait, `ProcessingStep`, `ProcessingStepResult` (internally tagged enum with `Completed`/`Failed`/`Skipped`), `ProcessingStepError`, and `ErrorKind` in `src/pipeline/mod.rs`. Includes 7 tests for JSON serialization shapes and `format_line()` rendering. Trait bound is `Serialize` (not `StepOutput` supertrait) — both bounds written explicitly where needed. +Implemented `StepOutput` trait, `ProcessingStep`, `ProcessingStepResult` +(internally tagged enum with `Completed`/`Failed`/`Skipped`), +`ProcessingStepError`, and `ErrorKind` in `src/pipeline/mod.rs`. Includes 7 +tests for JSON serialization shapes and `format_line()` rendering. Trait bound +is `Serialize` (not `StepOutput` supertrait) — both bounds written explicitly +where needed. diff --git a/docs/spec/todos/TODO-0080.md b/docs/spec/todos/TODO-0080.md index 33a086c..cec1482 100644 --- a/docs/spec/todos/TODO-0080.md +++ b/docs/spec/todos/TODO-0080.md @@ -27,8 +27,16 @@ files_updated: ## Summary -Define the output structs and run functions for each processing step in `src/pipeline/` submodules. Each module owns its run function, output struct, and `StepOutput` impl. +Define the output structs and run functions for each processing step in +`src/pipeline/` submodules. Each module owns its run function, output struct, +and `StepOutput` impl. ## Resolution -All step modules implemented incrementally across TODO-0081 through TODO-0087 as each command was reworked. Each module provides a `run_*()` function that takes inputs, calls domain logic, measures elapsed time, and returns `ProcessingStepResult`. Output structs implement `Serialize` and `StepOutput`. Steps are shared across commands as designed (e.g. `run_scan` used by check, build, init, update; `run_validate` used by check, build, init, update; etc.). +All step modules implemented incrementally across TODO-0081 through TODO-0087 as +each command was reworked. Each module provides a `run_*()` function that takes +inputs, calls domain logic, measures elapsed time, and returns +`ProcessingStepResult`. Output structs implement `Serialize` and +`StepOutput`. Steps are shared across commands as designed (e.g. `run_scan` used +by check, build, init, update; `run_validate` used by check, build, init, +update; etc.). diff --git a/docs/spec/todos/TODO-0081.md b/docs/spec/todos/TODO-0081.md index 5a0110f..4ab2a9a 100644 --- a/docs/spec/todos/TODO-0081.md +++ b/docs/spec/todos/TODO-0081.md @@ -16,8 +16,15 @@ files_updated: ## Summary -Rework the check command to use the pipeline model. Check is a simple 3-step pipeline. +Rework the check command to use the pipeline model. Check is a simple 3-step +pipeline. ## Resolution -Reworked `check::run()` to return `CheckCommandOutput` directly (no `anyhow::Result`). Added `CheckProcessOutput` with `read_config`, `scan`, `validate` step results. `CheckCommandOutput` has `process` + `result: Option`. Compact success delegates to `CheckResult::format_text()`, verbose adds step lines, failure shows steps up to failure point. Exit code 1 for violations, 2 for failed steps. Removed `glob` and `elapsed_ms` from `CheckResult` (timing is per-step now). +Reworked `check::run()` to return `CheckCommandOutput` directly (no +`anyhow::Result`). Added `CheckProcessOutput` with `read_config`, `scan`, +`validate` step results. `CheckCommandOutput` has `process` + +`result: Option`. Compact success delegates to +`CheckResult::format_text()`, verbose adds step lines, failure shows steps up to +failure point. Exit code 1 for violations, 2 for failed steps. Removed `glob` +and `elapsed_ms` from `CheckResult` (timing is per-step now). diff --git a/docs/spec/todos/TODO-0082.md b/docs/spec/todos/TODO-0082.md index 2a81220..d1dc1d9 100644 --- a/docs/spec/todos/TODO-0082.md +++ b/docs/spec/todos/TODO-0082.md @@ -23,8 +23,26 @@ files_updated: ## Summary -Rework the build command to use the pipeline model. Build is the most complex pipeline (7 steps). +Rework the build command to use the pipeline model. Build is the most complex +pipeline (7 steps). ## Resolution -Reworked `build::run()` to return `BuildCommandOutput` directly (no `anyhow::Result`). Pipeline is `read_config → scan → validate → classify → load_model → embed_files → write_index` (7 steps). Created 2 new shared step modules: `classify` (classifies files as new/edited/unchanged/removed for incremental builds), `write_index` (writes Parquet index). Extended `embed.rs` with `run_embed_files()` and `embed_file()` (moved from build.rs). Removed `BuildOutcome` enum, `FileClassification`, `FileToEmbed`, `classify_files()`, `load_embedder()`, `embed_file()` from build.rs — all moved to pipeline modules. `BuildFileDetail` moved to `write_index.rs`. Config mutation (fill missing sections, `--set-*` flags) stays as inline command logic; errors land on `scan` step as `Failed(User)`. Config change detection lands on `classify` as `Failed(User)`. Dimension mismatch lands on `embed_files` as `Failed(User)`. Validation violations → `check_result: Option`, remaining steps `Skipped`. Conditional model loading: classify shows 0 `needs_embedding` → `load_model` and `embed_files` `Skipped`. Updated init.rs and update.rs to handle `BuildCommandOutput` instead of `BuildOutcome`. Exit code 2 for failed steps, 1 for violations. +Reworked `build::run()` to return `BuildCommandOutput` directly (no +`anyhow::Result`). Pipeline is +`read_config → scan → validate → classify → load_model → embed_files → write_index` +(7 steps). Created 2 new shared step modules: `classify` (classifies files as +new/edited/unchanged/removed for incremental builds), `write_index` (writes +Parquet index). Extended `embed.rs` with `run_embed_files()` and `embed_file()` +(moved from build.rs). Removed `BuildOutcome` enum, `FileClassification`, +`FileToEmbed`, `classify_files()`, `load_embedder()`, `embed_file()` from +build.rs — all moved to pipeline modules. `BuildFileDetail` moved to +`write_index.rs`. Config mutation (fill missing sections, `--set-*` flags) stays +as inline command logic; errors land on `scan` step as `Failed(User)`. Config +change detection lands on `classify` as `Failed(User)`. Dimension mismatch lands +on `embed_files` as `Failed(User)`. Validation violations → +`check_result: Option`, remaining steps `Skipped`. Conditional +model loading: classify shows 0 `needs_embedding` → `load_model` and +`embed_files` `Skipped`. Updated init.rs and update.rs to handle +`BuildCommandOutput` instead of `BuildOutcome`. Exit code 2 for failed steps, 1 +for violations. diff --git a/docs/spec/todos/TODO-0083.md b/docs/spec/todos/TODO-0083.md index f34327b..299b316 100644 --- a/docs/spec/todos/TODO-0083.md +++ b/docs/spec/todos/TODO-0083.md @@ -24,8 +24,22 @@ files_updated: ## Summary -Rework the init command to use the pipeline model. Init flattens build steps into its own pipeline when auto_build is enabled. +Rework the init command to use the pipeline model. Init flattens build steps +into its own pipeline when auto_build is enabled. ## Resolution -Reworked `init::run()` to return `InitCommandOutput` directly (no `anyhow::Result`). Pipeline is `scan → infer → write_config [→ validate → classify → load_model → embed_files → write_index]` (8 steps). Created 2 new shared step modules: `infer` (wraps `InferredSchema::infer()`) and `write_config` (bundles `MdvsToml::from_inferred()` + `check_reserved_names()` + `write()`). Pre-checks land on scan step as `Failed(User)`: path not a directory, config exists without `--force`, build flags without `--auto-build`. No-files check lands on infer as `Failed(User)`. Reserved name collision lands on write_config as `Failed(User)`. Violations after init → `Failed(Application)` on validate (bug). dry_run: scan+infer complete, write_config through write_index `Skipped`, result returned with `dry_run: true`. Build steps always full rebuild (no existing index). No exit code 1 (no violations path), exit code 2 for failed steps. Updated all callers in build.rs, update.rs, info.rs, search.rs tests. +Reworked `init::run()` to return `InitCommandOutput` directly (no +`anyhow::Result`). Pipeline is +`scan → infer → write_config [→ validate → classify → load_model → embed_files → write_index]` +(8 steps). Created 2 new shared step modules: `infer` (wraps +`InferredSchema::infer()`) and `write_config` (bundles +`MdvsToml::from_inferred()` + `check_reserved_names()` + `write()`). Pre-checks +land on scan step as `Failed(User)`: path not a directory, config exists without +`--force`, build flags without `--auto-build`. No-files check lands on infer as +`Failed(User)`. Reserved name collision lands on write_config as `Failed(User)`. +Violations after init → `Failed(Application)` on validate (bug). dry_run: +scan+infer complete, write_config through write_index `Skipped`, result returned +with `dry_run: true`. Build steps always full rebuild (no existing index). No +exit code 1 (no violations path), exit code 2 for failed steps. Updated all +callers in build.rs, update.rs, info.rs, search.rs tests. diff --git a/docs/spec/todos/TODO-0084.md b/docs/spec/todos/TODO-0084.md index 17459c2..375d1a4 100644 --- a/docs/spec/todos/TODO-0084.md +++ b/docs/spec/todos/TODO-0084.md @@ -17,8 +17,25 @@ files_updated: ## Summary -Rework the update command to use the pipeline model. Update flattens build steps into its own pipeline when auto_build is enabled. +Rework the update command to use the pipeline model. Update flattens build steps +into its own pipeline when auto_build is enabled. ## Resolution -Reworked `update::run()` to return `UpdateCommandOutput` directly (no `anyhow::Result`). Pipeline is `read_config → scan → infer → write_config [→ validate → classify → load_model → embed_files → write_index]` (9 steps). Pre-checks land on read_config: `--reinfer` + `--reinfer-all` conflict, unknown reinfer field name, config read failure. No-files check lands on scan as `Failed(User)`. Reserved name collision lands on write_config as `Failed(User)`. write_config is inline (patches existing config's `[fields]` section, not `run_write_config` which is init-specific). dry_run OR no changes → write_config through write_index `Skipped`. Build steps call individual step functions directly (same as init) — `run_validate`, `run_classify`, `run_load_model`, `run_embed_files`, `run_write_index` — no black-box delegation to `build::run()`. Incremental by default: reads existing index for classification, config change detection via `detect_config_changes` (made `pub(crate)`), dimension checks before embedding. Violations after update → `Failed(Application)` on validate ("this is a bug"). No exit code 1 (violations = bug = exit 2). Updated main.rs dispatch to remove `?` and add exit code 2. All 242 tests pass. +Reworked `update::run()` to return `UpdateCommandOutput` directly (no +`anyhow::Result`). Pipeline is +`read_config → scan → infer → write_config [→ validate → classify → load_model → embed_files → write_index]` +(9 steps). Pre-checks land on read_config: `--reinfer` + `--reinfer-all` +conflict, unknown reinfer field name, config read failure. No-files check lands +on scan as `Failed(User)`. Reserved name collision lands on write_config as +`Failed(User)`. write_config is inline (patches existing config's `[fields]` +section, not `run_write_config` which is init-specific). dry_run OR no changes → +write_config through write_index `Skipped`. Build steps call individual step +functions directly (same as init) — `run_validate`, `run_classify`, +`run_load_model`, `run_embed_files`, `run_write_index` — no black-box delegation +to `build::run()`. Incremental by default: reads existing index for +classification, config change detection via `detect_config_changes` (made +`pub(crate)`), dimension checks before embedding. Violations after update → +`Failed(Application)` on validate ("this is a bug"). No exit code 1 (violations += bug = exit 2). Updated main.rs dispatch to remove `?` and add exit code 2. All +242 tests pass. diff --git a/docs/spec/todos/TODO-0085.md b/docs/spec/todos/TODO-0085.md index 5bb6ec3..b6fe113 100644 --- a/docs/spec/todos/TODO-0085.md +++ b/docs/spec/todos/TODO-0085.md @@ -25,4 +25,12 @@ Rework the search command to use the pipeline model. ## Resolution -Reworked `search::run()` to return `SearchCommandOutput` directly (no `anyhow::Result`). Pipeline is `read_config → read_index → load_model → embed_query → execute_search` (5 steps). Created 3 new shared step modules: `load_model` (loads embedding model from config), `embed` (embeds query string), `execute_search` (runs search via backend with `--where` quote validation). Pre-check errors (missing embedding_model, missing index, model mismatch) land on `load_model` step as `Failed(User)` to fail fast before expensive model loading. Removed `elapsed_ms` from `SearchResult`. Exit code 2 for failed steps. +Reworked `search::run()` to return `SearchCommandOutput` directly (no +`anyhow::Result`). Pipeline is +`read_config → read_index → load_model → embed_query → execute_search` (5 +steps). Created 3 new shared step modules: `load_model` (loads embedding model +from config), `embed` (embeds query string), `execute_search` (runs search via +backend with `--where` quote validation). Pre-check errors (missing +embedding_model, missing index, model mismatch) land on `load_model` step as +`Failed(User)` to fail fast before expensive model loading. Removed `elapsed_ms` +from `SearchResult`. Exit code 2 for failed steps. diff --git a/docs/spec/todos/TODO-0086.md b/docs/spec/todos/TODO-0086.md index 78db481..0023fd1 100644 --- a/docs/spec/todos/TODO-0086.md +++ b/docs/spec/todos/TODO-0086.md @@ -22,4 +22,9 @@ Rework the info command to use the pipeline model. ## Resolution -Reworked `info::run()` to return `InfoCommandOutput` directly (no `anyhow::Result`). Pipeline is `read_config → scan → read_index` (3 steps — scan needed for file/field counts). Created `read_index` pipeline step module with `run_read_index()`, `ReadIndexOutput`, and `IndexData`. "No index found" is a normal `Completed` output. Removed `glob` and `elapsed_ms` from `InfoResult`. Exit code 2 for failed steps. +Reworked `info::run()` to return `InfoCommandOutput` directly (no +`anyhow::Result`). Pipeline is `read_config → scan → read_index` (3 steps — scan +needed for file/field counts). Created `read_index` pipeline step module with +`run_read_index()`, `ReadIndexOutput`, and `IndexData`. "No index found" is a +normal `Completed` output. Removed `glob` and `elapsed_ms` from `InfoResult`. +Exit code 2 for failed steps. diff --git a/docs/spec/todos/TODO-0087.md b/docs/spec/todos/TODO-0087.md index 1aef080..5d83260 100644 --- a/docs/spec/todos/TODO-0087.md +++ b/docs/spec/todos/TODO-0087.md @@ -22,4 +22,8 @@ Rework the clean command to use the pipeline model. ## Resolution -Reworked `clean::run()` to return `CleanCommandOutput` directly (no `anyhow::Result`). Created `delete_index` pipeline step module with `run_delete_index()` and `DeleteIndexOutput`. "Nothing to clean" is a normal `Completed` output. Symlink check returns `Failed(User)`. Removed `elapsed_ms` from `CleanResult` (timing is per-step now). Exit code 2 for failed steps. +Reworked `clean::run()` to return `CleanCommandOutput` directly (no +`anyhow::Result`). Created `delete_index` pipeline step module with +`run_delete_index()` and `DeleteIndexOutput`. "Nothing to clean" is a normal +`Completed` output. Symlink check returns `Failed(User)`. Removed `elapsed_ms` +from `CleanResult` (timing is per-step now). Exit code 2 for failed steps. diff --git a/docs/spec/todos/TODO-0088.md b/docs/spec/todos/TODO-0088.md index f0ab783..6c2c9bb 100644 --- a/docs/spec/todos/TODO-0088.md +++ b/docs/spec/todos/TODO-0088.md @@ -12,30 +12,42 @@ blocks: [] ## Summary -Rework `main.rs` to handle the new pipeline-based command outputs with consistent error rendering and exit codes. +Rework `main.rs` to handle the new pipeline-based command outputs with +consistent error rendering and exit codes. ## Details ### Exit codes - **0**: pipeline completed, result present (success) -- **1**: pipeline completed, normal result with actionable findings (violations in check/build) +- **1**: pipeline completed, normal result with actionable findings (violations + in check/build) - **2**: user error (4xx) — bad input, missing config, model mismatch - **3**: application error (5xx) — I/O failure, internal error ### Error handling layers -1. **Inner (pipeline)**: all anticipated errors caught explicitly inside step functions, returned as `ProcessingStepResult::Failed`. This is the primary error handling path. -2. **Outer (anyhow)**: commands return `anyhow::Result<*CommandOutput>` as a last safety net for truly unexpected failures. Should ideally never fire. If it does, render as application error in the appropriate output format. +1. **Inner (pipeline)**: all anticipated errors caught explicitly inside step + functions, returned as `ProcessingStepResult::Failed`. This is the primary + error handling path. +2. **Outer (anyhow)**: commands return `anyhow::Result<*CommandOutput>` as a + last safety net for truly unexpected failures. Should ideally never fire. If + it does, render as application error in the appropriate output format. ### Rendering Each `*CommandOutput` implements `CommandOutput` (or a new version of it): -- `--output text`: render completed steps as lines (verbose or on failure), render result if present -- `--output json`: serialize the full `*CommandOutput` including process steps and result + +- `--output text`: render completed steps as lines (verbose or on failure), + render result if present +- `--output json`: serialize the full `*CommandOutput` including process steps + and result ### Changes -- Each command match arm receives `*CommandOutput` instead of current result types -- Exit code determined by inspecting the output (violations? failed step? success?) -- Top-level `anyhow::Result` catch converts unexpected errors to structured application error output +- Each command match arm receives `*CommandOutput` instead of current result + types +- Exit code determined by inspecting the output (violations? failed step? + success?) +- Top-level `anyhow::Result` catch converts unexpected errors to structured + application error output diff --git a/docs/spec/todos/TODO-0089.md b/docs/spec/todos/TODO-0089.md index 92de8d7..c177a0d 100644 --- a/docs/spec/todos/TODO-0089.md +++ b/docs/spec/todos/TODO-0089.md @@ -14,12 +14,19 @@ blocks: [] ## Summary -The `search` command never checks whether the index is up to date with the files on disk. If files have been added, removed, or edited since the last `mdvs build`, search silently returns results from the stale index. Add a warning so users know when they should rebuild. +The `search` command never checks whether the index is up to date with the files +on disk. If files have been added, removed, or edited since the last +`mdvs build`, search silently returns results from the stale index. Add a +warning so users know when they should rebuild. ## Original Scope -Three approaches were considered (file count, filename set, full hash comparison) for detecting stale indexes and warning the user before search. +Three approaches were considered (file count, filename set, full hash +comparison) for detecting stale indexes and warning the user before search. ## Resolution -Subsumed by [TODO-0099](TODO-0099.md). The auto-build pipeline redesign makes `search` auto-build by default, so the index is always fresh. Users who disable auto-build (`--no-build` or config) are explicitly choosing stale results, making a warning unnecessary. +Subsumed by [TODO-0099](TODO-0099.md). The auto-build pipeline redesign makes +`search` auto-build by default, so the index is always fresh. Users who disable +auto-build (`--no-build` or config) are explicitly choosing stale results, +making a warning unnecessary. diff --git a/docs/spec/todos/TODO-0090.md b/docs/spec/todos/TODO-0090.md index e4f6b35..6d10ac2 100644 --- a/docs/spec/todos/TODO-0090.md +++ b/docs/spec/todos/TODO-0090.md @@ -15,8 +15,16 @@ files_updated: ## Summary -Remove the `check_result: Option` field from `BuildCommandOutput`. The validate step's `ValidateOutput` already carries violation counts in the process output. Users who need full violation details should run `mdvs check` separately. +Remove the `check_result: Option` field from `BuildCommandOutput`. +The validate step's `ValidateOutput` already carries violation counts in the +process output. Users who need full violation details should run `mdvs check` +separately. ## Resolution -Removed `check_result: Option` from `BuildCommandOutput`. `has_violations()` now checks `process.validate` for `violation_count > 0`. `format_text` shows a short message ("Build aborted — N violation(s) found. Run `mdvs check` for details.") instead of rendering the full violations table. Cleaned up all 13 return sites in `run()` and 2 test assertions. `main.rs` exit code 1 logic unchanged — still uses `has_violations()`. +Removed `check_result: Option` from `BuildCommandOutput`. +`has_violations()` now checks `process.validate` for `violation_count > 0`. +`format_text` shows a short message ("Build aborted — N violation(s) found. Run +`mdvs check` for details.") instead of rendering the full violations table. +Cleaned up all 13 return sites in `run()` and 2 test assertions. `main.rs` exit +code 1 logic unchanged — still uses `has_violations()`. diff --git a/docs/spec/todos/TODO-0091.md b/docs/spec/todos/TODO-0091.md index f657c35..af265d0 100644 --- a/docs/spec/todos/TODO-0091.md +++ b/docs/spec/todos/TODO-0091.md @@ -13,10 +13,14 @@ blocks: [] ## Summary -Umbrella TODO defining consistent output rules across all commands for both text and JSON formats, based on verbose flag and error state. Split into two independent implementation TODOs: +Umbrella TODO defining consistent output rules across all commands for both text +and JSON formats, based on verbose flag and error state. Split into two +independent implementation TODOs: -- [TODO-0092](TODO-0092.md) — Compact JSON output (result-only when no errors and not verbose) -- [TODO-0093](TODO-0093.md) — Verbose text output (process step lines on success with `-v`) +- [TODO-0092](TODO-0092.md) — Compact JSON output (result-only when no errors + and not verbose) +- [TODO-0093](TODO-0093.md) — Verbose text output (process step lines on success + with `-v`) ## Resolution @@ -24,7 +28,8 @@ Both children completed. All 7 commands now follow the same output rules: - **JSON compact**: `{"result": {...}}` — result only - **JSON verbose**: `{"process": {...}, "result": {...}}` — full struct -- **JSON error**: `{"process": {...}, "result": null}` — full struct with null result +- **JSON error**: `{"process": {...}, "result": null}` — full struct with null + result - **Text compact**: result-derived content only - **Text verbose**: process step lines followed by result content - **Text error**: process steps up to the failure point diff --git a/docs/spec/todos/TODO-0092.md b/docs/spec/todos/TODO-0092.md index a3a195d..09e622e 100644 --- a/docs/spec/todos/TODO-0092.md +++ b/docs/spec/todos/TODO-0092.md @@ -22,8 +22,15 @@ files_updated: ## Summary -Add `format_json(verbose: bool)` to the `CommandOutput` trait so that compact JSON (`--output json` without `-v`) emits only the `result` key, while verbose or error states emit the full struct (process + result). +Add `format_json(verbose: bool)` to the `CommandOutput` trait so that compact +JSON (`--output json` without `-v`) emits only the `result` key, while verbose +or error states emit the full struct (process + result). ## Resolution -Added `format_json(verbose)` to the `CommandOutput` trait with a default implementation that serializes the full struct. Added `format_json_compact` helper in `output.rs` that omits `process` when `result.is_some() && !verbose`. All 7 `*CommandOutput` impls override `format_json` via the helper. Removed `skip_serializing_if` from `result` fields on init/build/update so error cases show `"result": null`. +Added `format_json(verbose)` to the `CommandOutput` trait with a default +implementation that serializes the full struct. Added `format_json_compact` +helper in `output.rs` that omits `process` when `result.is_some() && !verbose`. +All 7 `*CommandOutput` impls override `format_json` via the helper. Removed +`skip_serializing_if` from `result` fields on init/build/update so error cases +show `"result": null`. diff --git a/docs/spec/todos/TODO-0093.md b/docs/spec/todos/TODO-0093.md index 5f60973..a770d35 100644 --- a/docs/spec/todos/TODO-0093.md +++ b/docs/spec/todos/TODO-0093.md @@ -17,8 +17,14 @@ files_updated: ## Summary -Update `format_text` on each command so that verbose mode (`--output text -v`) renders process step lines (with elapsed times) before the result content. +Update `format_text` on each command so that verbose mode (`--output text -v`) +renders process step lines (with elapsed times) before the result content. ## Resolution -Updated `format_text` on init, build, and update to show process step lines in verbose mode (check, search, info, clean already had this). Replaced the `StepFormatLine` trait + loop error pattern with per-step `format_line()` calls matching the existing pattern in the other 4 commands. All 7 commands now follow the same three-branch structure: verbose success (step lines + result), compact success (result only), error (steps up to failure). +Updated `format_text` on init, build, and update to show process step lines in +verbose mode (check, search, info, clean already had this). Replaced the +`StepFormatLine` trait + loop error pattern with per-step `format_line()` calls +matching the existing pattern in the other 4 commands. All 7 commands now follow +the same three-branch structure: verbose success (step lines + result), compact +success (result only), error (steps up to failure). diff --git a/docs/spec/todos/TODO-0094.md b/docs/spec/todos/TODO-0094.md index f0a565e..59f0fe8 100644 --- a/docs/spec/todos/TODO-0094.md +++ b/docs/spec/todos/TODO-0094.md @@ -12,11 +12,16 @@ blocks: [] ## Summary -`scan.rs` silently skips files that exceed safety limits (nesting depth, field count, file size, unreadable files, symlink escapes). This is non-transparent — the user doesn't understand why a file's fields aren't showing up. Change these to hard errors so the scan step fails with a clear message naming the file and the limit exceeded. +`scan.rs` silently skips files that exceed safety limits (nesting depth, field +count, file size, unreadable files, symlink escapes). This is non-transparent — +the user doesn't understand why a file's fields aren't showing up. Change these +to hard errors so the scan step fails with a clear message naming the file and +the limit exceeded. ## Details -These `continue` sites in `src/discover/scan.rs` should become `return Err(anyhow!(...))`: +These `continue` sites in `src/discover/scan.rs` should become +`return Err(anyhow!(...))`: 1. **File outside root directory** (symlink escape) — line 77 2. **File exceeds 100MB limit** — line 94 @@ -25,6 +30,10 @@ These `continue` sites in `src/discover/scan.rs` should become `return Err(anyho 5. **Frontmatter exceeds field count limit** — line 140 6. **Frontmatter exceeds nesting depth limit** — line 148 -The glob mismatch (`continue` at line 82) and bare-file filtering (`continue` at lines 112, 153) are intentional filtering, not safety limits — leave those as-is. +The glob mismatch (`continue` at line 82) and bare-file filtering (`continue` at +lines 112, 153) are intentional filtering, not safety limits — leave those +as-is. -No changes needed downstream: `pipeline/scan.rs` already converts `Err` from `ScannedFiles::scan()` into `ProcessingStepResult::Failed`, which the pipeline and output layer handle automatically. +No changes needed downstream: `pipeline/scan.rs` already converts `Err` from +`ScannedFiles::scan()` into `ProcessingStepResult::Failed`, which the pipeline +and output layer handle automatically. diff --git a/docs/spec/todos/TODO-0095.md b/docs/spec/todos/TODO-0095.md index 7adaea8..d775674 100644 --- a/docs/spec/todos/TODO-0095.md +++ b/docs/spec/todos/TODO-0095.md @@ -14,7 +14,8 @@ files_created: [.github/workflows/book.yml] ## Summary -Add a GitHub Actions workflow that builds the mdBook site and deploys it to GitHub Pages on push to main. +Add a GitHub Actions workflow that builds the mdBook site and deploys it to +GitHub Pages on push to main. ## Details @@ -24,8 +25,10 @@ The workflow needs to: 2. Run `mdbook build book/` 3. Deploy `book/book/` to GitHub Pages -Use the `actions/deploy-pages` pattern (upload artifact + deploy). Consider caching the cargo install step for faster builds. +Use the `actions/deploy-pages` pattern (upload artifact + deploy). Consider +caching the cargo install step for faster builds. -The Mermaid JS files (`mermaid.min.js`, `mermaid-init.js`) are committed to the repo in `book/`, so they're available at build time without extra setup. +The Mermaid JS files (`mermaid.min.js`, `mermaid-init.js`) are committed to the +repo in `book/`, so they're available at build time without extra setup. Enable GitHub Pages in repo settings (Source: GitHub Actions). diff --git a/docs/spec/todos/TODO-0096.md b/docs/spec/todos/TODO-0096.md index dc0f539..7bd2b99 100644 --- a/docs/spec/todos/TODO-0096.md +++ b/docs/spec/todos/TODO-0096.md @@ -12,30 +12,38 @@ blocks: [] # TODO-0096: Change array type display from String[] to Array(String) > **Closeout (2026-05-12)** — Folded into [TODO-0097](./TODO-0097.md) step 7 -> (output rendering for object flattening). `FieldTypeSerde::Display` now -> emits the function-style form: `Array(String)` / `Array(Array(Integer))` -> / `Array(Object{time: String, value: Float})`. Object types render with -> an explicit `Object{...}` prefix for symmetry with `Array(...)`. Book -> snippets and prose still reference the old postfix form in some places; -> those are updated as part of TODO-0097 step 10 (specs + book). +> (output rendering for object flattening). `FieldTypeSerde::Display` now emits +> the function-style form: `Array(String)` / `Array(Array(Integer))` / +> `Array(Object{time: String, value: Float})`. Object types render with an +> explicit `Object{...}` prefix for symmetry with `Array(...)`. Book snippets +> and prose still reference the old postfix form in some places; those are +> updated as part of TODO-0097 step 10 (specs + book). ## Summary -The current display representation for array types uses a bracket suffix notation (`String[]`, `Integer[][]`) which is inflexible and inconsistent with how Object types are displayed (`{key: Type}`). Change to a parenthesized form like `Array(String)`, `Array(Array(Integer))` to match the Object pattern. +The current display representation for array types uses a bracket suffix +notation (`String[]`, `Integer[][]`) which is inflexible and inconsistent with +how Object types are displayed (`{key: Type}`). Change to a parenthesized form +like `Array(String)`, `Array(Array(Integer))` to match the Object pattern. ## Details -Current display (in `src/schema/shared.rs`, `impl fmt::Display for FieldTypeSerde`): +Current display (in `src/schema/shared.rs`, +`impl fmt::Display for FieldTypeSerde`): + - `String[]` for `Array(String)` - `Integer[][]` for `Array(Array(Integer))` - `{key: Type}` for Object -The bracket notation breaks down with complex nested types and doesn't visually match the Object display convention. A consistent approach would be: +The bracket notation breaks down with complex nested types and doesn't visually +match the Object display convention. A consistent approach would be: + - `Array(String)` instead of `String[]` - `Array(Array(Integer))` instead of `Integer[][]` - `Object({key: Type})` could also be considered for full consistency This affects: + - `src/schema/shared.rs` — `Display` impl for `FieldTypeSerde` - All command output that shows field types (init, update, info, check) - Book documentation (update type references) diff --git a/docs/spec/todos/TODO-0097.md b/docs/spec/todos/TODO-0097.md index 1ed93a0..5de6ea4 100644 --- a/docs/spec/todos/TODO-0097.md +++ b/docs/spec/todos/TODO-0097.md @@ -18,15 +18,15 @@ related: > **Closeout (2026-05-12)** — Wave C complete. All 12 steps shipped or > transparently completed; example_kb migrated (43 fields, dotted > `calibration.*` leaves); all docs swept for consistency. Folded in -> [TODO-0096](./TODO-0096.md) (function-style type display) at step 7. -> One follow-up TODO derived from Wave C: [TODO-0155](./TODO-0155.md) -> (reusable type definitions via `$defs` / `$ref`) — voluntary, medium -> priority. See the per-step closeout notes below for the actual landing -> mechanism of each step. +> [TODO-0096](./TODO-0096.md) (function-style type display) at step 7. One +> follow-up TODO derived from Wave C: [TODO-0155](./TODO-0155.md) (reusable type +> definitions via `$defs` / `$ref`) — voluntary, medium priority. See the +> per-step closeout notes below for the actual landing mechanism of each step. ## Summary -Replace the monolithic Object type representation with dot-separated leaf keys. Instead of one field `calibration` with type: +Replace the monolithic Object type representation with dot-separated leaf keys. +Instead of one field `calibration` with type: ```json { @@ -54,17 +54,25 @@ Represent it as individual leaf fields: The current Object type has several limitations: -1. **No per-leaf nullability** — `nullable` is a single boolean on the top-level field. With exploded keys, each leaf can be independently nullable. -2. **Hard to read** — nested Object types produce long wrapped strings in table output (e.g., `{adjusted: {intensity: Float, wavelength: Float}, baseline: {intensity: Float, ...}}`). -3. **Inconsistent query syntax** — `--where` uses bracket notation for nested access (`calibration['baseline']['wavelength']`), while dot-separated keys would allow natural dot notation. -4. **No per-leaf validation** — can't set different `allowed`/`required` constraints on individual nested keys. -5. **Inflexible** — adding a new nested key in one file affects the entire Object type, making the toml representation harder to manage. +1. **No per-leaf nullability** — `nullable` is a single boolean on the top-level + field. With exploded keys, each leaf can be independently nullable. +2. **Hard to read** — nested Object types produce long wrapped strings in table + output (e.g., + `{adjusted: {intensity: Float, wavelength: Float}, baseline: {intensity: Float, ...}}`). +3. **Inconsistent query syntax** — `--where` uses bracket notation for nested + access (`calibration['baseline']['wavelength']`), while dot-separated keys + would allow natural dot notation. +4. **No per-leaf validation** — can't set different `allowed`/`required` + constraints on individual nested keys. +5. **Inflexible** — adding a new nested key in one file affects the entire + Object type, making the toml representation harder to manage. ## Details ### In `mdvs.toml` Before: + ```toml [[fields.field]] name = "calibration" @@ -74,6 +82,7 @@ nullable = false ``` After: + ```toml [[fields.field]] name = "calibration.adjusted.intensity" @@ -108,132 +117,245 @@ nullable = false ### Impact areas -- **Inference** (`src/discover/`): flatten Object types into dot-separated paths during schema inference -- **Config** (`src/schema/config.rs`): `TomlField` names can now contain dots; no more `FieldTypeSerde::Object` variant (or it becomes unused) -- **Validation** (`src/cmd/check.rs`): navigate into nested YAML values using dot-separated paths -- **Storage** (`src/index/storage.rs`): may still use Arrow Struct columns internally, but field names in the schema map to dot paths -- **Output** (`src/output.rs`): simpler display — each leaf is a scalar type, no nested type rendering +- **Inference** (`src/discover/`): flatten Object types into dot-separated paths + during schema inference +- **Config** (`src/schema/config.rs`): `TomlField` names can now contain dots; + no more `FieldTypeSerde::Object` variant (or it becomes unused) +- **Validation** (`src/cmd/check.rs`): navigate into nested YAML values using + dot-separated paths +- **Storage** (`src/index/storage.rs`): may still use Arrow Struct columns + internally, but field names in the schema map to dot paths +- **Output** (`src/output.rs`): simpler display — each leaf is a scalar type, no + nested type rendering - **`--where`**: dot notation for nested access instead of bracket notation -- **Display format**: the `FieldTypeSerde` Display impl for Object becomes unnecessary; each field is just a scalar or array type +- **Display format**: the `FieldTypeSerde` Display impl for Object becomes + unnecessary; each field is just a scalar or array type ## Scope clarifications (decided 2026-05-11) ### What gets flattened, what doesn't -- **Top-level `FieldType::Object` is flattened** into dotted-name leaf fields. After this lands, `type = "Object"` or `type = { object = {...} }` on a `[[fields.field]]` is **rejected at config load**. -- **`FieldType::Object` *inside* `Array` is NOT flattened** at the Wave C boundary. `type = { array = { object = { time = "String", value = "Float" } } }` was kept as a valid inline shape here. Per-element nullability and per-element path-scoping don't compose (array indices are not file-system paths), so flattening Array-of-Object would be semantically wrong. -- **`FieldType::Object` is therefore retained in the Rust enum** — it's no longer a valid top-level type, but it remains an internal building block (Wave C uses it for the synthesized storage tree). - -> **Closeout note (TODO-0155, 2026-05-13):** `Array(Object{...})` was subsequently rejected entirely at parse, inference, and config-load. The "reusable definitions" follow-up that originally lived here moved to [TODO-0155](./TODO-0155.md) for the on-disk type vocabulary unification, and the open question of a first-class Array-of-structured-item representation lives in [TODO-0156](./TODO-0156.md). The v0 workaround is parallel scalar arrays. +- **Top-level `FieldType::Object` is flattened** into dotted-name leaf fields. + After this lands, `type = "Object"` or `type = { object = {...} }` on a + `[[fields.field]]` is **rejected at config load**. +- **`FieldType::Object` _inside_ `Array` is NOT flattened** at the Wave C + boundary. + `type = { array = { object = { time = "String", value = "Float" } } }` was + kept as a valid inline shape here. Per-element nullability and per-element + path-scoping don't compose (array indices are not file-system paths), so + flattening Array-of-Object would be semantically wrong. +- **`FieldType::Object` is therefore retained in the Rust enum** — it's no + longer a valid top-level type, but it remains an internal building block (Wave + C uses it for the synthesized storage tree). + +> **Closeout note (TODO-0155, 2026-05-13):** `Array(Object{...})` was +> subsequently rejected entirely at parse, inference, and config-load. The +> "reusable definitions" follow-up that originally lived here moved to +> [TODO-0155](./TODO-0155.md) for the on-disk type vocabulary unification, and +> the open question of a first-class Array-of-structured-item representation +> lives in [TODO-0156](./TODO-0156.md). The v0 workaround is parallel scalar +> arrays. ### Where the flattening lives -| Layer | Representation | -|---|---| -| Source YAML frontmatter | Nested objects (untouched: `calibration: { baseline: { wavelength: 850.0 } }`) | -| `mdvs.toml` | Flat list of `[[fields.field]]` with dotted names (`name = "calibration.baseline.wavelength"`) | +| Layer | Representation | +| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Source YAML frontmatter | Nested objects (untouched: `calibration: { baseline: { wavelength: 850.0 } }`) | +| `mdvs.toml` | Flat list of `[[fields.field]]` with dotted names (`name = "calibration.baseline.wavelength"`) | | Canonical JSON Schema (`dsl_to_canonical`) | **Nested** `properties` tree (`properties.calibration.properties.baseline.properties.wavelength`) — the translator reconstructs the nesting from dotted names | -| `jsonschema::Validator` input | Nested (matches the YAML's natural shape) | -| Parquet `data` Struct column | Nested Arrow Struct (matches YAML / canonical schema) — unchanged | -| `--where` queries | Dot notation (`calibration.baseline.wavelength = 850`) via `[search.aliases]` mapping to nested struct access | +| `jsonschema::Validator` input | Nested (matches the YAML's natural shape) | +| Parquet `data` Struct column | Nested Arrow Struct (matches YAML / canonical schema) — unchanged | +| `--where` queries | Dot notation (`calibration.baseline.wavelength = 850`) via `[search.aliases]` mapping to nested struct access | -The flattening is a **mdvs.toml UX choice**, not a representational one. Storage and validation both operate on the natural nested shape. The translator (`dsl_to_canonical`) bridges flat-toml to nested-schema; `canonical_to_dsl` reverses it. +The flattening is a **mdvs.toml UX choice**, not a representational one. Storage +and validation both operate on the natural nested shape. The translator +(`dsl_to_canonical`) bridges flat-toml to nested-schema; `canonical_to_dsl` +reverses it. ### Field name rules -- A field name MAY contain `.` to indicate nested-leaf membership. Dotted names are parsed as paths during translation. +- A field name MAY contain `.` to indicate nested-leaf membership. Dotted names + are parsed as paths during translation. - A field name MUST NOT start or end with `.`, nor contain `..`. -- A frontmatter key that literally contains a `.` in YAML (rare but legal) is treated as a flat key for inference. If both `foo.bar` (literal YAML key) and `foo: { bar: ... }` (nested) appear across files for the "same" path, config validation reports a collision. +- A frontmatter key that literally contains a `.` in YAML (rare but legal) is + treated as a flat key for inference. If both `foo.bar` (literal YAML key) and + `foo: { bar: ... }` (nested) appear across files for the "same" path, config + validation reports a collision. ## Execution plan -Step numbering is for reference, not strict dependency order. The dependency graph is at the end. +Step numbering is for reference, not strict dependency order. The dependency +graph is at the end. ### Step 1 — Inference: emit one InferredField per leaf **Prereqs:** none. **Files touched:** -- `crates/mdvs/src/discover/infer/types.rs` — `infer_field_types` recurses into nested `Value::Object` and tracks each leaf path separately. The `FieldTypeInfo` map keyed by dotted name. -- `crates/mdvs/src/discover/infer/paths.rs` — path inference per leaf (each leaf gets its own `allowed`/`required` glob set from the files where it appeared). -- `crates/mdvs/src/discover/infer/mod.rs` — `InferredField` already has scalar `name: String`; that name now contains dots for leaves. `InferredSchema` emits one entry per leaf. -- `crates/mdvs/src/discover/field_type.rs` — `FieldType::Object` widening is **only used for Array-of-Object inner types** now. Top-level recursion stops at the Object boundary (the recursion produces leaf entries instead of returning an Object value). + +- `crates/mdvs/src/discover/infer/types.rs` — `infer_field_types` recurses into + nested `Value::Object` and tracks each leaf path separately. The + `FieldTypeInfo` map keyed by dotted name. +- `crates/mdvs/src/discover/infer/paths.rs` — path inference per leaf (each leaf + gets its own `allowed`/`required` glob set from the files where it appeared). +- `crates/mdvs/src/discover/infer/mod.rs` — `InferredField` already has scalar + `name: String`; that name now contains dots for leaves. `InferredSchema` emits + one entry per leaf. +- `crates/mdvs/src/discover/field_type.rs` — `FieldType::Object` widening is + **only used for Array-of-Object inner types** now. Top-level recursion stops + at the Object boundary (the recursion produces leaf entries instead of + returning an Object value). **Tests:** -- New: inference of `calibration: { baseline: { wavelength: 850.0 } }` across N files produces five separate `InferredField`s with dotted names; each carries its own widened type, observed_types, and path globs. -- Existing widening tests: top-level Object widening tests are reshaped or moved into Array-of-Object inner-widening tests. -- Edge case: Object observed only in some files, scalar in others, on the same top-level key — current behavior is "widen to String"; needs an explicit decision (likely error: the key can't be both a leaf and an object). -**DoD:** `cargo test -p mdvs` green; inference on `example_kb` (which has `calibration`) produces leaf fields with names like `calibration.baseline.wavelength`. +- New: inference of `calibration: { baseline: { wavelength: 850.0 } }` across N + files produces five separate `InferredField`s with dotted names; each carries + its own widened type, observed_types, and path globs. +- Existing widening tests: top-level Object widening tests are reshaped or moved + into Array-of-Object inner-widening tests. +- Edge case: Object observed only in some files, scalar in others, on the same + top-level key — current behavior is "widen to String"; needs an explicit + decision (likely error: the key can't be both a leaf and an object). + +**DoD:** `cargo test -p mdvs` green; inference on `example_kb` (which has +`calibration`) produces leaf fields with names like +`calibration.baseline.wavelength`. ### Step 2 — Config: reject top-level Object, accept dotted names -**Prereqs:** none (can land before step 1, but step 1 needs it to write configs). +**Prereqs:** none (can land before step 1, but step 1 needs it to write +configs). **Files touched:** -- `crates/mdvs/src/schema/config.rs` — `MdvsToml::validate()` adds a sixth invariant: `[[fields.field]]` with `type = Object` (or `{ object = ... }`) is rejected at config load. The error message tells the user to flatten into dotted-name leaf fields. -- `crates/mdvs/src/schema/shared.rs` — `FieldTypeSerde::Object` variant kept (still valid inside `Array`), but the load-time gate enforces no-top-level. -- Validation of dotted field names: well-formed (no leading/trailing dot, no `..`, no whitespace around the dots). + +- `crates/mdvs/src/schema/config.rs` — `MdvsToml::validate()` adds a sixth + invariant: `[[fields.field]]` with `type = Object` (or `{ object = ... }`) is + rejected at config load. The error message tells the user to flatten into + dotted-name leaf fields. +- `crates/mdvs/src/schema/shared.rs` — `FieldTypeSerde::Object` variant kept + (still valid inside `Array`), but the load-time gate enforces no-top-level. +- Validation of dotted field names: well-formed (no leading/trailing dot, no + `..`, no whitespace around the dots). **Tests:** -- Inline tests: load an mdvs.toml with top-level `type = "Object"` → reject with a clear message. Load with `type = { array = { object = {...} } }` → accept. + +- Inline tests: load an mdvs.toml with top-level `type = "Object"` → reject with + a clear message. Load with `type = { array = { object = {...} } }` → accept. - Dotted-name well-formedness tests. -**DoD:** the gate rejects top-level Object configs and accepts dotted-name leaf configs. +**DoD:** the gate rejects top-level Object configs and accepts dotted-name leaf +configs. ### Step 3 — Translator: dotted names → nested JSON Schema **Prereqs:** step 2. **Files touched:** -- `crates/mdvs/src/schema/json_schema.rs::dsl_to_canonical` — group `[[fields.field]]` by their dotted name prefix and reconstruct the nested `properties` tree. Each leaf field becomes a property at its appropriate depth. `x-mdvs.allowed` / `x-mdvs.required` / `x-mdvs.preprocess` move to the leaf property's level. -- `canonical_to_dsl` — the reverse: walk the nested `properties` tree, emit one `[[fields.field]]` per leaf, dot-join the path to form the name. Read back `x-mdvs.*` per leaf. -- Remove the `FieldType::Object` arm placeholder from `type_subschema` (it was a knowing placeholder for top-level; top-level is now gone). The Object arm survives only when called recursively for Array-of-Object inner types — keep that path but tighten it: emit `{type: "object", properties: {...}}` instead of `additionalProperties: true`. + +- `crates/mdvs/src/schema/json_schema.rs::dsl_to_canonical` — group + `[[fields.field]]` by their dotted name prefix and reconstruct the nested + `properties` tree. Each leaf field becomes a property at its appropriate + depth. `x-mdvs.allowed` / `x-mdvs.required` / `x-mdvs.preprocess` move to the + leaf property's level. +- `canonical_to_dsl` — the reverse: walk the nested `properties` tree, emit one + `[[fields.field]]` per leaf, dot-join the path to form the name. Read back + `x-mdvs.*` per leaf. +- Remove the `FieldType::Object` arm placeholder from `type_subschema` (it was a + knowing placeholder for top-level; top-level is now gone). The Object arm + survives only when called recursively for Array-of-Object inner types — keep + that path but tighten it: emit `{type: "object", properties: {...}}` instead + of `additionalProperties: true`. **Tests:** -- Round-trip: an mdvs.toml with `calibration.baseline.wavelength` + `calibration.adjusted.wavelength` translates to a nested `properties.calibration.properties.baseline.properties.wavelength` schema and reverses back. -- Round-trip with mixed leaves (`title` flat + `calibration.*` dotted) produces a flat-at-top, nested-in-calibration schema. -- Array-of-Object round-trip: `type = { array = { object = { x = "Integer" } } }` survives the trip. -**DoD:** `dsl_to_canonical(canonical_to_dsl(s)) == s` for every config shape we currently produce, plus dotted-name examples. +- Round-trip: an mdvs.toml with `calibration.baseline.wavelength` + + `calibration.adjusted.wavelength` translates to a nested + `properties.calibration.properties.baseline.properties.wavelength` schema and + reverses back. +- Round-trip with mixed leaves (`title` flat + `calibration.*` dotted) produces + a flat-at-top, nested-in-calibration schema. +- Array-of-Object round-trip: + `type = { array = { object = { x = "Integer" } } }` survives the trip. + +**DoD:** `dsl_to_canonical(canonical_to_dsl(s)) == s` for every config shape we +currently produce, plus dotted-name examples. ### Step 4 — Validation: per-leaf validators on nested data **Prereqs:** steps 1, 2, 3. **Files touched:** -- `crates/mdvs/src/cmd/check.rs::FieldValidators` — today this is `HashMap<&str, Validator>` keyed by field name. With nested validation, jsonschema operates on the **whole frontmatter** against the nested schema, not per-field. Either: - - **(a)** Compile one root `Validator` from `dsl_to_canonical(config)` and validate the entire frontmatter object once per file. Error mapping uses `instance_path` to recover the dotted field name. - - **(b)** Per-field validators keyed by dotted name; navigate the YAML value at that path before validating against the leaf schema. - **Recommended: (a)** — closer to how `jsonschema` is designed to work; one compile per validate-call; cleaner error mapping. The current per-field-validator structure was a Wave B convenience; this step replaces it. - -- `crates/mdvs/src/cmd/check.rs::check_field_values` — iterate `[[fields.field]]` entries, navigate YAML by dotted name to extract the leaf value, run path-scoping checks (Disallowed) per leaf. The jsonschema engine handles per-leaf type/range/length/pattern via the nested schema. -- `crates/mdvs/src/cmd/check.rs::check_required_fields` — `MissingRequired` checks navigate YAML by dotted name. Absent intermediate objects count as the leaf being absent. -- `crates/mdvs/src/preprocess.rs::Pipeline` — preprocessors apply per-leaf. The pipeline is keyed by dotted name. `strict_subtype_check` operates on the leaf value. +- `crates/mdvs/src/cmd/check.rs::FieldValidators` — today this is + `HashMap<&str, Validator>` keyed by field name. With nested validation, + jsonschema operates on the **whole frontmatter** against the nested schema, + not per-field. Either: + - **(a)** Compile one root `Validator` from `dsl_to_canonical(config)` and + validate the entire frontmatter object once per file. Error mapping uses + `instance_path` to recover the dotted field name. + - **(b)** Per-field validators keyed by dotted name; navigate the YAML value + at that path before validating against the leaf schema. + + **Recommended: (a)** — closer to how `jsonschema` is designed to work; one + compile per validate-call; cleaner error mapping. The current + per-field-validator structure was a Wave B convenience; this step replaces it. + +- `crates/mdvs/src/cmd/check.rs::check_field_values` — iterate + `[[fields.field]]` entries, navigate YAML by dotted name to extract the leaf + value, run path-scoping checks (Disallowed) per leaf. The jsonschema engine + handles per-leaf type/range/length/pattern via the nested schema. +- `crates/mdvs/src/cmd/check.rs::check_required_fields` — `MissingRequired` + checks navigate YAML by dotted name. Absent intermediate objects count as the + leaf being absent. +- `crates/mdvs/src/preprocess.rs::Pipeline` — preprocessors apply per-leaf. The + pipeline is keyed by dotted name. `strict_subtype_check` operates on the leaf + value. **Tests:** -- All existing check tests stay green after adapting Object-using tests to dotted form. -- New: nested data, per-leaf type violation reports the dotted leaf name in `field` and the right `WrongType` detail. -- New: per-leaf `nullable = false` rejects null at that leaf; per-leaf `nullable = true` passes. -- New: per-leaf `required = ["projects/alpha/**"]` triggers `MissingRequired` when a file in that path lacks the leaf (even if the parent object is present without that key). -**DoD:** `cargo test -p mdvs` green; check produces per-leaf violations correctly. +- All existing check tests stay green after adapting Object-using tests to + dotted form. +- New: nested data, per-leaf type violation reports the dotted leaf name in + `field` and the right `WrongType` detail. +- New: per-leaf `nullable = false` rejects null at that leaf; per-leaf + `nullable = true` passes. +- New: per-leaf `required = ["projects/alpha/**"]` triggers `MissingRequired` + when a file in that path lacks the leaf (even if the parent object is present + without that key). + +**DoD:** `cargo test -p mdvs` green; check produces per-leaf violations +correctly. ### Step 5 — Storage: keep nested Arrow Structs, alias dotted names **Prereqs:** step 3 (the canonical schema describes the nested storage shape). **Files touched:** -- `crates/mdvs/src/index/storage.rs::build_array` — already recursive on `FieldType::Object`. Step 3 removed top-level Object from configs; the recursion now runs only for inner shapes that the translator reconstructed. **Refactor:** `build_files_batch` builds the `data` Struct from the **canonical schema** rather than walking `TomlField`s directly. Single source of truth for nesting. -- `crates/mdvs/src/index/storage.rs` — `COL_DATA` Struct children mirror the schema's nested `properties` tree. Field names in the schema are dotted but the Arrow column names are single-segment (the dot is split at translation time). -- `crates/mdvs/src/search.rs` — `SearchContext` builds the search view by aliasing dotted names to nested struct access (`data['calibration']['baseline']['wavelength'] AS "calibration.baseline.wavelength"`). The existing `[search.aliases]` machinery handles user-facing aliasing; this is the system-side equivalent. + +- `crates/mdvs/src/index/storage.rs::build_array` — already recursive on + `FieldType::Object`. Step 3 removed top-level Object from configs; the + recursion now runs only for inner shapes that the translator reconstructed. + **Refactor:** `build_files_batch` builds the `data` Struct from the + **canonical schema** rather than walking `TomlField`s directly. Single source + of truth for nesting. +- `crates/mdvs/src/index/storage.rs` — `COL_DATA` Struct children mirror the + schema's nested `properties` tree. Field names in the schema are dotted but + the Arrow column names are single-segment (the dot is split at translation + time). +- `crates/mdvs/src/search.rs` — `SearchContext` builds the search view by + aliasing dotted names to nested struct access + (`data['calibration']['baseline']['wavelength'] AS "calibration.baseline.wavelength"`). + The existing `[search.aliases]` machinery handles user-facing aliasing; this + is the system-side equivalent. **Tests:** -- Build + search round-trip on a fixture with nested calibration data; `--where calibration.baseline.wavelength > 800` works. + +- Build + search round-trip on a fixture with nested calibration data; + `--where calibration.baseline.wavelength > 800` works. - `mdvs info` shows leaf fields with dotted names and scalar types. -**DoD:** index builds, search works with dot-notation `--where` against leaf fields. +**DoD:** index builds, search works with dot-notation `--where` against leaf +fields. ### Step 6 — `--where` dot notation @@ -241,67 +363,90 @@ Step numbering is for reference, not strict dependency order. The dependency gra > **Closeout note (2026-05-12)** — Step 6 completed transparently as a > side-effect of step 5. The originally-planned auto-aliasing layer -> (`data['cal']['base']['wave'] AS "cal.base.wave"` flat top-level -> projections) turned out to be unnecessary: step 5's nested Arrow -> Struct storage combined with DataFusion's native SQL struct-field -> access (and `SearchContext::new`'s existing auto-promotion of `data` -> Struct children to top-level aliases) means a user can write -> `WHERE cal.baseline.wavelength > 800` without any new alias machinery. -> Bracket notation (`data['cal']['baseline']['wavelength']`) stays +> (`data['cal']['base']['wave'] AS "cal.base.wave"` flat top-level projections) +> turned out to be unnecessary: step 5's nested Arrow Struct storage combined +> with DataFusion's native SQL struct-field access (and `SearchContext::new`'s +> existing auto-promotion of `data` Struct children to top-level aliases) means +> a user can write `WHERE cal.baseline.wavelength > 800` without any new alias +> machinery. Bracket notation (`data['cal']['baseline']['wavelength']`) stays > available as an alternative. > -> Collision detection: the existing check in `SearchContext::new` -> operates on top-level Struct child names (single-segment) and remains -> correct. Literal-dot YAML keys are rejected at scan time (per the -> open-questions resolution), so no new collision class exists. +> Collision detection: the existing check in `SearchContext::new` operates on +> top-level Struct child names (single-segment) and remains correct. Literal-dot +> YAML keys are rejected at scan time (per the open-questions resolution), so no +> new collision class exists. > > Two regression tests added in `crates/mdvs/src/search.rs`: > `dotted_leaf_where_clause` and `dotted_leaf_where_clause_via_bracket_syntax`. **Files touched (historic plan, kept for reference):** -- `crates/mdvs/src/search.rs` — view construction generates aliases `data['calibration']['baseline']['wavelength'] AS "calibration.baseline.wavelength"` for every leaf. User-facing `--where` accepts the dotted form natively. -- Aliases collision detection: a top-level scalar field named `calibration.baseline.wavelength` (literal dot in YAML key) would collide with the nested alias. Detect at view build and error out. + +- `crates/mdvs/src/search.rs` — view construction generates aliases + `data['calibration']['baseline']['wavelength'] AS "calibration.baseline.wavelength"` + for every leaf. User-facing `--where` accepts the dotted form natively. +- Aliases collision detection: a top-level scalar field named + `calibration.baseline.wavelength` (literal dot in YAML key) would collide with + the nested alias. Detect at view build and error out. **Tests:** -- `mdvs search "query" --where "calibration.baseline.wavelength = 850"` works against built example_kb. -- Bracket notation continues to work as a fallback (back-compat with anything else querying `data['calibration']`). -**DoD:** dot notation works in `--where`; collision detection rejects ambiguous configs. +- `mdvs search "query" --where "calibration.baseline.wavelength = 850"` works + against built example_kb. +- Bracket notation continues to work as a fallback (back-compat with anything + else querying `data['calibration']`). + +**DoD:** dot notation works in `--where`; collision detection rejects ambiguous +configs. ### Step 7 — Output rendering: each leaf as its own row **Prereqs:** steps 1, 2. **Files touched:** -- `crates/mdvs/src/output.rs` — `DiscoveredField` already takes a `field_type: String` display string. After flattening, leaves render as scalars (`Float`, `String[]`, etc.); no more nested type-tree rendering. -- `crates/mdvs/src/schema/shared.rs::FieldTypeSerde::Display` — Object arm produces output only when nested inside Array. Top-level Object display is gone. -- `crates/mdvs/src/cmd/init.rs`, `update.rs`, `info.rs` — verify the discovered-field tables look right post-flattening (each leaf is its own table block). + +- `crates/mdvs/src/output.rs` — `DiscoveredField` already takes a + `field_type: String` display string. After flattening, leaves render as + scalars (`Float`, `String[]`, etc.); no more nested type-tree rendering. +- `crates/mdvs/src/schema/shared.rs::FieldTypeSerde::Display` — Object arm + produces output only when nested inside Array. Top-level Object display is + gone. +- `crates/mdvs/src/cmd/init.rs`, `update.rs`, `info.rs` — verify the + discovered-field tables look right post-flattening (each leaf is its own table + block). **Tests:** -- Snapshot/visual verification: `info example_kb` renders all calibration leaves as separate rows; no nested type string. -**DoD:** every leaf is rendered independently; no nested type strings in normal output. +- Snapshot/visual verification: `info example_kb` renders all calibration leaves + as separate rows; no nested type string. + +**DoD:** every leaf is rendered independently; no nested type strings in normal +output. ### Step 8 — Inference for Array-of-Object retention **Prereqs:** step 1. -> **Closeout (2026-05-12)** — DoD met as a side-effect of step 1's -> design. `collect_leaves` stops at `Value::Array` (treats arrays as -> leaves regardless of element content), and the Array's inner element -> type flows through the existing `FieldType::from_widen` / -> `FieldType::from(&Value)` path that handled Array(Object) pre-Wave-C. -> No new production code was needed. +> **Closeout (2026-05-12)** — DoD met as a side-effect of step 1's design. +> `collect_leaves` stops at `Value::Array` (treats arrays as leaves regardless +> of element content), and the Array's inner element type flows through the +> existing `FieldType::from_widen` / `FieldType::from(&Value)` path that handled +> Array(Object) pre-Wave-C. No new production code was needed. > > Three tests in `crates/mdvs/src/discover/infer/types.rs` cover this: > `array_of_object_stays_inline_not_exploded` (added in step 1), -> `array_of_object_widens_inner_keys_across_files` (cross-file inner -> key union), `array_of_object_widens_inner_scalar_types` (mixed inner -> scalar types collapse to String). +> `array_of_object_widens_inner_keys_across_files` (cross-file inner key union), +> `array_of_object_widens_inner_scalar_types` (mixed inner scalar types collapse +> to String). **Files touched:** -- `crates/mdvs/src/discover/infer/types.rs` — when widening encounters `Value::Array` whose elements are objects, the inferred type stays `Array(Object{...})`. The Object-inside-Array uses the existing `FieldType::Object` widening logic (kept around as inner-only). -- Test: inference of `readings: [{ time: "..", value: 0.5 }, { time: "..", value: 0.6 }]` produces a single `Array(Object{time: String, value: Float})` field, not flattened. + +- `crates/mdvs/src/discover/infer/types.rs` — when widening encounters + `Value::Array` whose elements are objects, the inferred type stays + `Array(Object{...})`. The Object-inside-Array uses the existing + `FieldType::Object` widening logic (kept around as inner-only). +- Test: inference of + `readings: [{ time: "..", value: 0.5 }, { time: "..", value: 0.6 }]` produces + a single `Array(Object{time: String, value: Float})` field, not flattened. **DoD:** Array-of-Object inference produces inline-struct types. @@ -309,17 +454,16 @@ Step numbering is for reference, not strict dependency order. The dependency gra **Prereqs:** steps 1–7. -> **Closeout (2026-05-12)** — example_kb was migrated incrementally -> during steps 1, 4, 5, 6, 7 verification runs. Final state, verified -> end-to-end: +> **Closeout (2026-05-12)** — example_kb was migrated incrementally during steps +> 1, 4, 5, 6, 7 verification runs. Final state, verified end-to-end: > -> | Command | Result | -> |---|---| -> | `init --force example_kb` | 43 fields (was 39 pre-Wave-C: −1 `calibration` Object, +5 dotted leaves) | -> | `check example_kb --no-update` | zero violations | -> | `build example_kb --force` | 43 files, 59 chunks, nested Arrow Struct shape | -> | `search "..." --where "calibration.baseline.wavelength > 800"` | one hit (`projects/alpha/notes/experiment-1.md`) | -> | `info example_kb` | renders `Array(String)` / `Object{...}` types | +> | Command | Result | +> | -------------------------------------------------------------- | ------------------------------------------------------------------------ | +> | `init --force example_kb` | 43 fields (was 39 pre-Wave-C: −1 `calibration` Object, +5 dotted leaves) | +> | `check example_kb --no-update` | zero violations | +> | `build example_kb --force` | 43 files, 59 chunks, nested Arrow Struct shape | +> | `search "..." --where "calibration.baseline.wavelength > 800"` | one hit (`projects/alpha/notes/experiment-1.md`) | +> | `info example_kb` | renders `Array(String)` / `Object{...}` types | > > Calibration is now five dotted-name `[[fields.field]]` entries: > @@ -341,44 +485,72 @@ Step numbering is for reference, not strict dependency order. The dependency gra > # ... > ``` > -> The `Array(Object{...})` demo was **deferred**. Wave C's unit tests -> (steps 1, 5, 6, 8) cover Array(Object) inference, storage, search, -> and widening comprehensively, so example_kb doesn't need the -> additional content to demonstrate the feature. Adding it later is -> a small additive change (e.g., a `samples: [{timestamp, intensity}]` -> field on a handful of experiment files). +> The `Array(Object{...})` demo was **deferred**. Wave C's unit tests (steps 1, +> 5, 6, 8) cover Array(Object) inference, storage, search, and widening +> comprehensively, so example_kb doesn't need the additional content to +> demonstrate the feature. Adding it later is a small additive change (e.g., a +> `samples: [{timestamp, intensity}]` field on a handful of experiment files). **Files touched:** -- `example_kb/mdvs.toml` — regenerate via `mdvs init --force example_kb`. The `calibration` field is replaced by five dotted-name leaf entries. + +- `example_kb/mdvs.toml` — regenerate via `mdvs init --force example_kb`. The + `calibration` field is replaced by five dotted-name leaf entries. - Verify `check` returns zero violations. - Verify `build`, `search`, `info`, `clean` work end-to-end. -- Optionally: add a `readings` field as an `Array(Object{time, value})` demonstrating retained inline structs — only if it fits the running example. +- Optionally: add a `readings` field as an `Array(Object{time, value})` + demonstrating retained inline structs — only if it fits the running example. -**DoD:** example_kb's mdvs.toml uses dotted leaves; all commands work; book screenshots / snippets reflect the new state. +**DoD:** example_kb's mdvs.toml uses dotted leaves; all commands work; book +screenshots / snippets reflect the new state. ### Step 10 — Spec + book updates **Prereqs:** steps 1–9. -> **Closeout (2026-05-12)** — docs swept for Wave C consistency. Three layered changes applied: +> **Closeout (2026-05-12)** — docs swept for Wave C consistency. Three layered +> changes applied: > -> 1. **Display refresh** (TODO-0096 folded in): every `String[]` / `Integer[]` style reference replaced with `Array(String)` / `Array(Integer)`. Object display gets the `Object{...}` prefix. -> 2. **Dotted-name leaf flattening**: top-level Object is rejected; nested YAML structure is expressed as dotted-name leaves in `mdvs.toml`. New section in `architecture.md` ("Dotted-name leaf flattening") documents the three-layer model (flat toml ↔ nested canonical schema ↔ nested Arrow Struct). Concepts/types.md rewritten — old Object-key-merging section replaced with a leaf-flattening walkthrough showing the calibration example landing as five dotted leaves. -> 3. **Invariant count**: `MdvsToml::validate()` now lists eight invariants (was five in Wave B). Updated in `architecture.md`, `AGENTS.md`. +> 1. **Display refresh** (TODO-0096 folded in): every `String[]` / `Integer[]` +> style reference replaced with `Array(String)` / `Array(Integer)`. Object +> display gets the `Object{...}` prefix. +> 2. **Dotted-name leaf flattening**: top-level Object is rejected; nested YAML +> structure is expressed as dotted-name leaves in `mdvs.toml`. New section in +> `architecture.md` ("Dotted-name leaf flattening") documents the three-layer +> model (flat toml ↔ nested canonical schema ↔ nested Arrow Struct). +> Concepts/types.md rewritten — old Object-key-merging section replaced with +> a leaf-flattening walkthrough showing the calibration example landing as +> five dotted leaves. +> 3. **Invariant count**: `MdvsToml::validate()` now lists eight invariants (was +> five in Wave B). Updated in `architecture.md`, `AGENTS.md`. > -> Files updated: `docs/spec/{architecture, shared, storage}.md`, `docs/spec/commands/{check, search}.md`, `book/src/{introduction, configuration, search-guide}.md`, `book/src/concepts/{types, schema, validation}.md`, `book/src/commands/{info, init}.md`, `book/src/recipes/obsidian.md`, `README.md`, `AGENTS.md`. +> Files updated: `docs/spec/{architecture, shared, storage}.md`, +> `docs/spec/commands/{check, search}.md`, +> `book/src/{introduction, configuration, search-guide}.md`, +> `book/src/concepts/{types, schema, validation}.md`, +> `book/src/commands/{info, init}.md`, `book/src/recipes/obsidian.md`, +> `README.md`, `AGENTS.md`. > -> mdBook builds clean. Example_kb output snippets show 43-field counts and `Array(String)` rendering. +> mdBook builds clean. Example_kb output snippets show 43-field counts and +> `Array(String)` rendering. **Files touched:** -- `docs/spec/architecture.md` — describe the flat-toml / nested-schema layering; update the Constraint Architecture section if it mentions Object-as-top-level. -- `docs/spec/shared.md` — `FieldTypeSerde` doc reflects "Object only inside Array". -- `docs/spec/storage.md` — `data` Struct column structure description (no change in shape, but the rationale is now "matches canonical schema" not "matches Object FieldType"). -- `docs/spec/commands/check.md`, `init.md`, `update.md`, `build.md` — type examples use dotted leaves. -- `book/src/concepts/types.md` — replace top-level Object section with the flattening rule; explain when Array-of-Object stays inline. -- `book/src/concepts/schema.md` — discovered fields render as scalars/arrays only at the leaf level. + +- `docs/spec/architecture.md` — describe the flat-toml / nested-schema layering; + update the Constraint Architecture section if it mentions Object-as-top-level. +- `docs/spec/shared.md` — `FieldTypeSerde` doc reflects "Object only inside + Array". +- `docs/spec/storage.md` — `data` Struct column structure description (no change + in shape, but the rationale is now "matches canonical schema" not "matches + Object FieldType"). +- `docs/spec/commands/check.md`, `init.md`, `update.md`, `build.md` — type + examples use dotted leaves. +- `book/src/concepts/types.md` — replace top-level Object section with the + flattening rule; explain when Array-of-Object stays inline. +- `book/src/concepts/schema.md` — discovered fields render as scalars/arrays + only at the leaf level. - `book/src/concepts/validation.md` — per-leaf validation explained. -- `book/src/configuration.md` — TOML reference; remove top-level Object syntax; document dotted names. +- `book/src/configuration.md` — TOML reference; remove top-level Object syntax; + document dotted names. - `book/src/search-guide.md` — dot notation in `--where`. - `book/src/commands/*.md` — output snippets refresh. - `README.md` — feature bullets if any mention nested objects. @@ -391,36 +563,45 @@ Step numbering is for reference, not strict dependency order. The dependency gra **Prereqs:** step 3. > **Closeout (2026-05-12)** — landed transparently in step 3. The -> `type_subschema` Object arm now emits `{type: "object", -> additionalProperties: true, properties: {...}}` from -> `FieldType::Object`'s children (only reached via Array(Object) after -> invariant 6 rejects top-level Object). The permissive Wave-B -> placeholder is gone. Verified by `array_of_object_emits_items_properties` -> in `crates/mdvs/src/schema/json_schema.rs::tests`. +> `type_subschema` Object arm now emits +> `{type: "object", additionalProperties: true, properties: {...}}` from +> `FieldType::Object`'s children (only reached via Array(Object) after invariant +> 6 rejects top-level Object). The permissive Wave-B placeholder is gone. +> Verified by `array_of_object_emits_items_properties` in +> `crates/mdvs/src/schema/json_schema.rs::tests`. **Files touched:** -- `crates/mdvs/src/schema/json_schema.rs::type_subschema` — the Object arm's "knowing placeholder" (`additionalProperties: true`) is replaced by a proper emit using the recovered nested structure. After step 3 it's only reached via Array-of-Object inner types; tighten it to emit `{type: object, properties: {...}}` based on the inner shape. -**DoD:** Object arm in the translator emits proper structured schemas, no permissive fallback. +- `crates/mdvs/src/schema/json_schema.rs::type_subschema` — the Object arm's + "knowing placeholder" (`additionalProperties: true`) is replaced by a proper + emit using the recovered nested structure. After step 3 it's only reached via + Array-of-Object inner types; tighten it to emit + `{type: object, properties: {...}}` based on the inner shape. + +**DoD:** Object arm in the translator emits proper structured schemas, no +permissive fallback. ### Step 12 — Memory + TODO bookkeeping **Prereqs:** all of the above. -> **Closeout (2026-05-12)** — this step. TODO-0097 marked done. -> TODO-0149 marked done (all three waves complete, with step 13 -> deferred to TODO-0154). Index updated. Memory updated (`MEMORY.md` -> and `todo_149_status.md`) — Wave C design decisions captured: flat -> mdvs.toml ↔ nested canonical schema ↔ nested Arrow Struct three-layer -> model; function-style type display; Array(Object) inline retention. -> Stale Wave-B-era "Object validation is intentionally loose placeholder" -> notes removed. +> **Closeout (2026-05-12)** — this step. TODO-0097 marked done. TODO-0149 marked +> done (all three waves complete, with step 13 deferred to TODO-0154). Index +> updated. Memory updated (`MEMORY.md` and `todo_149_status.md`) — Wave C design +> decisions captured: flat mdvs.toml ↔ nested canonical schema ↔ nested Arrow +> Struct three-layer model; function-style type display; Array(Object) inline +> retention. Stale Wave-B-era "Object validation is intentionally loose +> placeholder" notes removed. **Files touched:** -- `docs/spec/todos/TODO-0149.md` — Wave C section marked complete (object flattening half). + +- `docs/spec/todos/TODO-0149.md` — Wave C section marked complete (object + flattening half). - `docs/spec/todos/TODO-0097.md` (this file) — status to done. - `docs/spec/todos/index.md` — flip statuses. -- Memory updates: drop the "Object validation is intentionally loose placeholder" notes; capture the dotted-name + nested-schema layering as a design decision. +- Memory updates: drop the "Object validation is intentionally loose + placeholder" notes; capture the dotted-name + nested-schema layering as a + design decision. **DoD:** TODO-0097 and TODO-0149's Wave C (object-flattening half) closed. @@ -434,25 +615,58 @@ Step numbering is for reference, not strict dependency order. The dependency gra └─→ 11 ─────────────┘ ``` -Steps 1, 2, 8 are foundation. Step 3 (translator) sits on top of config (2). Steps 4, 5, 6 form the validation-storage-query chain. Step 7 (output) can land in parallel with 4–6. Step 9 (example_kb migration) is the integration milestone. Step 10 (docs) and 11 (translator cleanup) close out before step 12 (bookkeeping). +Steps 1, 2, 8 are foundation. Step 3 (translator) sits on top of config (2). +Steps 4, 5, 6 form the validation-storage-query chain. Step 7 (output) can land +in parallel with 4–6. Step 9 (example_kb migration) is the integration +milestone. Step 10 (docs) and 11 (translator cleanup) close out before step 12 +(bookkeeping). ## Resolved design decisions -1. **Auto-aliasing for nested leaves** — non-issue. `[search.aliases]` aliases only mdvs's internal columns (per the error message at `search.rs:200`), not frontmatter fields. The view's existing auto-promotion (`data['title'] AS title`) extends naturally to dotted paths in step 6. No user-controlled alias for frontmatter fields exists, so there's no precedence to settle. Combined with #3 below, the alias space has no ambiguity. - -2. **Empty Object `{}`** — accepted, zero leaves. If a leaf path is `{}` in some files and populated in others, the populated leaves get inferred and the empty-object files contribute "absent" for required-checks (same as a partial top-level scalar). If *all* files have `{}`, no leaves are emitted and there's nothing to validate at that path. Object cardinality (`minProperties`) is a future feature, not a Wave C concern. - -3. **Literal-dot YAML keys** (`"foo.bar": "..."` as a flat YAML key) — **reject at scan time** with a clear error. A frontmatter key containing `.` conflicts with mdvs's nesting convention. Implementation: extend the frontmatter-error handling (Wave B's silent-drop fix) with a new error message: *"frontmatter key 'foo.bar' contains '.', which conflicts with mdvs's nested-field convention; use nested mapping syntax or a Stage 1 preprocessor (TODO when shipped) to remap"*. The error surfaces as `FrontmatterUnrepresentable` at the document level. - - **Future hook:** when the Stage 1 field-name preprocessor lands (currently `FieldNameStage` is an empty enum in `preprocess.rs`), it will expose a `RenameKey` / `EscapeDots` stage so users who genuinely need literal-dot keys can opt in. Note this in the error message *only after* Stage 1 ships — until then, the error stays generic. - -4. **Dotted CLI args** (`mdvs update reinfer calibration.baseline.wavelength`) — works as-is. Clap doesn't constrain string content; the lookup is by string equality. Add one test case to lock it in. - -5. **JSON Schema export verbosity** — out of scope. The nested `properties` shape is standard JSON Schema and required for interop with any compliant validator. One-line note in the book. +1. **Auto-aliasing for nested leaves** — non-issue. `[search.aliases]` aliases + only mdvs's internal columns (per the error message at `search.rs:200`), not + frontmatter fields. The view's existing auto-promotion + (`data['title'] AS title`) extends naturally to dotted paths in step 6. No + user-controlled alias for frontmatter fields exists, so there's no precedence + to settle. Combined with #3 below, the alias space has no ambiguity. + +2. **Empty Object `{}`** — accepted, zero leaves. If a leaf path is `{}` in some + files and populated in others, the populated leaves get inferred and the + empty-object files contribute "absent" for required-checks (same as a partial + top-level scalar). If _all_ files have `{}`, no leaves are emitted and + there's nothing to validate at that path. Object cardinality + (`minProperties`) is a future feature, not a Wave C concern. + +3. **Literal-dot YAML keys** (`"foo.bar": "..."` as a flat YAML key) — **reject + at scan time** with a clear error. A frontmatter key containing `.` conflicts + with mdvs's nesting convention. Implementation: extend the frontmatter-error + handling (Wave B's silent-drop fix) with a new error message: _"frontmatter + key 'foo.bar' contains '.', which conflicts with mdvs's nested-field + convention; use nested mapping syntax or a Stage 1 preprocessor (TODO when + shipped) to remap"_. The error surfaces as `FrontmatterUnrepresentable` at + the document level. + + **Future hook:** when the Stage 1 field-name preprocessor lands (currently + `FieldNameStage` is an empty enum in `preprocess.rs`), it will expose a + `RenameKey` / `EscapeDots` stage so users who genuinely need literal-dot keys + can opt in. Note this in the error message _only after_ Stage 1 ships — until + then, the error stays generic. + +4. **Dotted CLI args** (`mdvs update reinfer calibration.baseline.wavelength`) — + works as-is. Clap doesn't constrain string content; the lookup is by string + equality. Add one test case to lock it in. + +5. **JSON Schema export verbosity** — out of scope. The nested `properties` + shape is standard JSON Schema and required for interop with any compliant + validator. One-line note in the book. ## Out of scope - Reusable definitions (`$defs` / `$ref`) — see [TODO-0155](./TODO-0155.md). -- Type-name renames (lowercase / JSON Schema names) — decided against (see TODO-0149 closeout note). -- Auto-creating definitions when inference detects repeated Array(Object) shapes — see TODO-0155's "auto-extraction" out-of-scope note. -- Per-leaf categorical inference for Object children — already covered by step 1's per-leaf inference loop (each leaf flows through the existing categorical heuristic independently). +- Type-name renames (lowercase / JSON Schema names) — decided against (see + TODO-0149 closeout note). +- Auto-creating definitions when inference detects repeated Array(Object) shapes + — see TODO-0155's "auto-extraction" out-of-scope note. +- Per-leaf categorical inference for Object children — already covered by step + 1's per-leaf inference loop (each leaf flows through the existing categorical + heuristic independently). diff --git a/docs/spec/todos/TODO-0098.md b/docs/spec/todos/TODO-0098.md index 6a096ac..535febd 100644 --- a/docs/spec/todos/TODO-0098.md +++ b/docs/spec/todos/TODO-0098.md @@ -14,7 +14,10 @@ files_updated: [src/cmd/build.rs, src/cmd/update.rs, book/src/commands/build.md] ## Summary -When switching to a model with different embedding dimensions, `mdvs build --force` fails with a dimension mismatch error even though `--force` should trigger a full rebuild. The user must run `mdvs clean` first, then rebuild. +When switching to a model with different embedding dimensions, +`mdvs build --force` fails with a dimension mismatch error even though `--force` +should trigger a full rebuild. The user must run `mdvs clean` first, then +rebuild. ## Details @@ -36,9 +39,12 @@ mdvs build example_kb --set-model minishlab/potion-base-2M --force ### Expected behavior -`--force` should handle the dimension change — it's already doing a full rebuild, so it should discard the old index entirely rather than comparing dimensions against it. +`--force` should handle the dimension change — it's already doing a full +rebuild, so it should discard the old index entirely rather than comparing +dimensions against it. ### Impact areas -- `src/cmd/build.rs` — dimension check happens before the rebuild starts; should be skipped or deferred when `--force` is set +- `src/cmd/build.rs` — dimension check happens before the rebuild starts; should + be skipped or deferred when `--force` is set - `src/index/storage.rs` — dimension validation during chunk retention diff --git a/docs/spec/todos/TODO-0099.md b/docs/spec/todos/TODO-0099.md index efc1386..5f01f72 100644 --- a/docs/spec/todos/TODO-0099.md +++ b/docs/spec/todos/TODO-0099.md @@ -15,14 +15,22 @@ subsumed: ## Summary -Restructure how commands chain into each other. The principle: **downstream commands can auto-run upstream steps, not the other way around.** The dependency chain is `update → build → search`, so only downstream commands (`check`, `build`, `search`) offer auto-inclusion of upstream steps. Remove auto-build from both `init` and `update`. +Restructure how commands chain into each other. The principle: **downstream +commands can auto-run upstream steps, not the other way around.** The dependency +chain is `update → build → search`, so only downstream commands (`check`, +`build`, `search`) offer auto-inclusion of upstream steps. Remove auto-build +from both `init` and `update`. ## Current State -- `init`: auto-builds by default, `--suppress-auto-build` to skip. Also accepts `--model`, `--revision`, `--chunk-size` for the embedded build step. Duplicates the entire build pipeline inline (~400 lines). -- `update`: `[update].auto_build = true` by default, `--build` flag to override. Also duplicates the build pipeline inline. +- `init`: auto-builds by default, `--suppress-auto-build` to skip. Also accepts + `--model`, `--revision`, `--chunk-size` for the embedded build step. + Duplicates the entire build pipeline inline (~400 lines). +- `update`: `[update].auto_build = true` by default, `--build` flag to override. + Also duplicates the build pipeline inline. - `build`: no auto-update, assumes schema is current. -- `search`: no auto-update, no auto-build, queries whatever's in `.mdvs/`. Stale index is silent. +- `search`: no auto-update, no auto-build, queries whatever's in `.mdvs/`. Stale + index is silent. - `check`: no auto-update, validates against whatever's in `mdvs.toml`. ## Design @@ -34,15 +42,24 @@ update (schema) → build (index) → search (query) upstream downstream ``` -Each command can optionally run the steps above it in the chain. No command auto-runs steps below it. +Each command can optionally run the steps above it in the chain. No command +auto-runs steps below it. ### Design principles -1. **Downstream auto-runs upstream, never the reverse.** Init and update never trigger build. Build and search can trigger update. Search can trigger build. -2. **Auto step failure = command failure.** If auto-update fails inside build, build aborts. If auto-build fails inside search, search aborts. No fallback to stale data. -3. **Explicit config, conservative defaults.** Commands write explicit `auto_update = true` / `auto_build = true` when creating config sections. If a key is missing from the toml (hand-deleted), the default is `false`. No hidden convenience — the toml is the source of truth. -4. **No interactive prompts.** Informational messages only (stdout, part of the output structure). -5. **`--no-*` flags for one-shot overrides.** `--no-update` and `--no-build` suppress auto steps for a single invocation without changing the toml. +1. **Downstream auto-runs upstream, never the reverse.** Init and update never + trigger build. Build and search can trigger update. Search can trigger build. +2. **Auto step failure = command failure.** If auto-update fails inside build, + build aborts. If auto-build fails inside search, search aborts. No fallback + to stale data. +3. **Explicit config, conservative defaults.** Commands write explicit + `auto_update = true` / `auto_build = true` when creating config sections. If + a key is missing from the toml (hand-deleted), the default is `false`. No + hidden convenience — the toml is the source of truth. +4. **No interactive prompts.** Informational messages only (stdout, part of the + output structure). +5. **`--no-*` flags for one-shot overrides.** `--no-update` and `--no-build` + suppress auto steps for a single invocation without changing the toml. ### Command changes @@ -51,38 +68,49 @@ Each command can optionally run the steps above it in the chain. No command auto **Becomes schema-only.** Scan → infer → write `mdvs.toml`. Config sections written by init: + - `[scan]` — always - `[check]` — always (new section, `auto_update = true`) - `[fields]` — always - `[update]` — hidden (empty, kept as placeholder via `skip_serializing_if`) Config sections **not** written by init: -- `[embedding_model]`, `[chunking]`, `[build]`, `[search]` — created by first `build` run + +- `[embedding_model]`, `[chunking]`, `[build]`, `[search]` — created by first + `build` run Flags removed: + - `--suppress-auto-build` (no longer applicable) - `--model` (belongs to `build --set-model`) - `--revision` (belongs to `build --set-revision`) - `--chunk-size` (belongs to `build --set-chunk-size`) Flags kept: + - `--force` — behavior tightened (see below) - `--dry-run` - `--glob` - `--ignore-bare-files` -**`--force` behavior change:** Today init only checks for `mdvs.toml`. New behavior: +**`--force` behavior change:** Today init only checks for `mdvs.toml`. New +behavior: + - `init` on a clean directory (no `mdvs.toml`, no `.mdvs/`) → works -- `init` on a directory with `mdvs.toml` OR `.mdvs/` → error: "mdvs is already initialized (use --force to reinitialize)" -- `init --force` → delete both `mdvs.toml` and `.mdvs/` if they exist, then proceed with fresh init +- `init` on a directory with `mdvs.toml` OR `.mdvs/` → error: "mdvs is already + initialized (use --force to reinitialize)" +- `init --force` → delete both `mdvs.toml` and `.mdvs/` if they exist, then + proceed with fresh init -**No model download during init.** Users who only need validation (`init` + `check`) never touch embeddings. +**No model download during init.** Users who only need validation (`init` + +`check`) never touch embeddings. #### `check` **Gains optional auto-update** via new `[check]` section. New config: + ```toml [check] auto_update = true @@ -90,20 +118,25 @@ auto_update = true New flag: `--no-update` to suppress auto-update for that invocation. -Default behavior: `check` → auto-update (scan + infer + merge fields + write toml) → validate. -With `--no-update`: validate only (current behavior). +Default behavior: `check` → auto-update (scan + infer + merge fields + write +toml) → validate. With `--no-update`: validate only (current behavior). #### `update` -**Becomes pure inference.** No auto-build. Remove `[update].auto_build` and `--build` flag. +**Becomes pure inference.** No auto-build. Remove `[update].auto_build` and +`--build` flag. -`[update]` section becomes empty. Kept in code as `UpdateConfig` (placeholder for future settings) but hidden from toml serialization via `skip_serializing_if` when empty. If someone hand-edits to add keys, they survive serialization. +`[update]` section becomes empty. Kept in code as `UpdateConfig` (placeholder +for future settings) but hidden from toml serialization via +`skip_serializing_if` when empty. If someone hand-edits to add keys, they +survive serialization. #### `build` **Gains auto-update** via new `[build]` section. New config: + ```toml [build] auto_update = true @@ -111,16 +144,20 @@ auto_update = true New flag: `--no-update` to suppress auto-update for that invocation. -Default behavior: `build` → auto-update (scan + infer + merge fields + write toml) → validate → classify → embed → write index. -With `--no-update`: read config → scan → validate → classify → embed → write index (current behavior). +Default behavior: `build` → auto-update (scan + infer + merge fields + write +toml) → validate → classify → embed → write index. With `--no-update`: read +config → scan → validate → classify → embed → write index (current behavior). -`[build]` section is optional — created on first `build` run alongside `[embedding_model]` and `[chunking]`. When absent, `auto_update` defaults to `false` (conservative). +`[build]` section is optional — created on first `build` run alongside +`[embedding_model]` and `[chunking]`. When absent, `auto_update` defaults to +`false` (conservative). #### `search` **Gains auto-update and auto-build** on `[search]` section. New config keys: + ```toml [search] default_limit = 10 @@ -130,31 +167,38 @@ auto_build = true New flags: `--no-update`, `--no-build` to suppress for that invocation. -Default behavior: `search` → auto-update → auto-build → query. -With `--no-build`: auto-update → query existing index. -With `--no-update`: auto-build (with current schema) → query. -With `--no-update --no-build`: query only (current behavior). +Default behavior: `search` → auto-update → auto-build → query. With +`--no-build`: auto-update → query existing index. With `--no-update`: auto-build +(with current schema) → query. With `--no-update --no-build`: query only +(current behavior). -`[search]` section is optional — created on first `build` run. When absent, both `auto_update` and `auto_build` default to `false`. +`[search]` section is optional — created on first `build` run. When absent, both +`auto_update` and `auto_build` default to `false`. -**First-time search with no index:** Search detects no `.mdvs/` and auto-build is enabled → triggers build. An informational message is rendered as part of the stdout output (not stderr): "No index found. Building index (this may take a moment)...". This message is part of the output structure, rendered in both text and JSON formats. See TODO-0110 for the recursive output architecture that supports this. +**First-time search with no index:** Search detects no `.mdvs/` and auto-build +is enabled → triggers build. An informational message is rendered as part of the +stdout output (not stderr): "No index found. Building index (this may take a +moment)...". This message is part of the output structure, rendered in both text +and JSON formats. See TODO-0110 for the recursive output architecture that +supports this. -**Auto-build failure:** If build fails (e.g., validation errors), search aborts. No fallback to stale index. +**Auto-build failure:** If build fails (e.g., validation errors), search aborts. +No fallback to stale index. ### Combination matrix -| Command | `auto_update` | `auto_build` | Pipeline | -|---|---|---|---| -| `init` | — | — | scan → infer → write toml | -| `update` | — | — | read config → scan → infer → merge fields → write toml | -| `check` | true (default) | — | update → validate | -| `check --no-update` | false | — | read config → scan → validate | -| `build` | true (default) | — | update → validate → classify → embed → write index | -| `build --no-update` | false | — | read config → scan → validate → classify → embed → write index | -| `search` | true (default) | true (default) | update → build → query | -| `search --no-build` | true | false | update → query existing index | -| `search --no-update` | false | true | build → query | -| `search --no-update --no-build` | false | false | query only | +| Command | `auto_update` | `auto_build` | Pipeline | +| ------------------------------- | -------------- | -------------- | -------------------------------------------------------------- | +| `init` | — | — | scan → infer → write toml | +| `update` | — | — | read config → scan → infer → merge fields → write toml | +| `check` | true (default) | — | update → validate | +| `check --no-update` | false | — | read config → scan → validate | +| `build` | true (default) | — | update → validate → classify → embed → write index | +| `build --no-update` | false | — | read config → scan → validate → classify → embed → write index | +| `search` | true (default) | true (default) | update → build → query | +| `search --no-build` | true | false | update → query existing index | +| `search --no-update` | false | true | build → query | +| `search --no-update --no-build` | false | false | query only | ### Config changes @@ -176,7 +220,8 @@ auto_update = true auto_update = true ``` -- Optional — created on first `build` alongside `[embedding_model]` and `[chunking]` +- Optional — created on first `build` alongside `[embedding_model]` and + `[chunking]` - If absent, `auto_update` defaults to `false` - `BuildConfig` struct in `config.rs` @@ -184,23 +229,26 @@ auto_update = true - Remove `auto_build` field - `UpdateConfig` becomes empty -- Hidden from toml via `skip_serializing_if` (serialized only if non-empty, for future fields) -- Existing toml files with `[update].auto_build` will get a hard error from `deny_unknown_fields` (TODO-0111) +- Hidden from toml via `skip_serializing_if` (serialized only if non-empty, for + future fields) +- Existing toml files with `[update].auto_build` will get a hard error from + `deny_unknown_fields` (TODO-0111) #### `[search]` section - Add `auto_update: bool` and `auto_build: bool` -- Always serialized when the section exists (no `skip_serializing_if` on these keys) +- Always serialized when the section exists (no `skip_serializing_if` on these + keys) - If the section is absent, both default to `false` #### Default rules summary -| Section | Written by | Optional? | Missing key default | -|---|---|---|---| -| `[check]` | `init` | yes | `auto_update = false` | -| `[build]` | first `build` | yes | `auto_update = false` | -| `[search]` | first `build` | yes | `auto_update = false`, `auto_build = false` | -| `[update]` | — | hidden | empty, placeholder | +| Section | Written by | Optional? | Missing key default | +| ---------- | ------------- | --------- | ------------------------------------------- | +| `[check]` | `init` | yes | `auto_update = false` | +| `[build]` | first `build` | yes | `auto_update = false` | +| `[search]` | first `build` | yes | `auto_update = false`, `auto_build = false` | +| `[update]` | — | hidden | empty, placeholder | ### User workflows @@ -236,39 +284,61 @@ mdvs search "query" example_kb # query (auto-build see mdvs check --no-update example_kb # validate against committed mdvs.toml, no schema changes ``` -CI recipe in `book/src/recipes/ci.md` should emphasize `--no-update` or setting `[check].auto_update = false` in the committed toml for reproducible validation. +CI recipe in `book/src/recipes/ci.md` should emphasize `--no-update` or setting +`[check].auto_update = false` in the committed toml for reproducible validation. ## Impact ### Code changes -- `src/cmd/init.rs` — remove entire build pipeline (~400 lines), remove `--suppress-auto-build`/`--model`/`--revision`/`--chunk-size` flags, tighten `--force` to check for `.mdvs/`, simplify `InitProcessOutput` (remove validate/classify/load_model/embed_files/write_index steps), update `InitResult` (remove `build_result`) -- `src/cmd/update.rs` — remove entire build pipeline, remove `--build` flag, simplify `UpdateProcessOutput` and `UpdateResult` -- `src/cmd/check.rs` — add auto-update step, add `--no-update` flag, update `CheckProcessOutput` -- `src/cmd/build.rs` — add auto-update step, add `--no-update` flag, update `BuildProcessOutput` -- `src/cmd/search.rs` — add auto-update + auto-build steps, add `--no-update`/`--no-build` flags, update `SearchProcessOutput` -- `src/schema/config.rs` — add `CheckConfig`, add `BuildConfig`, empty `UpdateConfig` (hidden), add `auto_update`/`auto_build` to `SearchConfig`, add `[check]` to `MdvsToml`, add `[build]` to `MdvsToml` +- `src/cmd/init.rs` — remove entire build pipeline (~400 lines), remove + `--suppress-auto-build`/`--model`/`--revision`/`--chunk-size` flags, tighten + `--force` to check for `.mdvs/`, simplify `InitProcessOutput` (remove + validate/classify/load_model/embed_files/write_index steps), update + `InitResult` (remove `build_result`) +- `src/cmd/update.rs` — remove entire build pipeline, remove `--build` flag, + simplify `UpdateProcessOutput` and `UpdateResult` +- `src/cmd/check.rs` — add auto-update step, add `--no-update` flag, update + `CheckProcessOutput` +- `src/cmd/build.rs` — add auto-update step, add `--no-update` flag, update + `BuildProcessOutput` +- `src/cmd/search.rs` — add auto-update + auto-build steps, add + `--no-update`/`--no-build` flags, update `SearchProcessOutput` +- `src/schema/config.rs` — add `CheckConfig`, add `BuildConfig`, empty + `UpdateConfig` (hidden), add `auto_update`/`auto_build` to `SearchConfig`, add + `[check]` to `MdvsToml`, add `[build]` to `MdvsToml` - `src/main.rs` — update clap definitions for all affected commands - Tests across all affected command modules ### Config migration -Existing `mdvs.toml` files with `[update].auto_build = true` will fail with a hard error from `deny_unknown_fields` (TODO-0111, prerequisite). Users must remove the `[update]` section or the `auto_build` key. This is intentional — the tool should not silently ignore stale config. +Existing `mdvs.toml` files with `[update].auto_build = true` will fail with a +hard error from `deny_unknown_fields` (TODO-0111, prerequisite). Users must +remove the `[update]` section or the `auto_build` key. This is intentional — the +tool should not silently ignore stale config. -New toml files written by `init` will not have `[embedding_model]`, `[chunking]`, `[build]`, or `[search]` sections until first `build` run. +New toml files written by `init` will not have `[embedding_model]`, +`[chunking]`, `[build]`, or `[search]` sections until first `build` run. ### Subsumed TODOs -- **TODO-0089** (stale index warning): No longer needed — `search` auto-builds by default, so the index is always fresh. Users who disable auto-build (`--no-build` or config) are explicitly choosing stale results. +- **TODO-0089** (stale index warning): No longer needed — `search` auto-builds + by default, so the index is always fresh. Users who disable auto-build + (`--no-build` or config) are explicitly choosing stale results. ### Documentation -- `book/src/commands/init.md` — remove auto-build section and build flags, document new `--force` behavior +- `book/src/commands/init.md` — remove auto-build section and build flags, + document new `--force` behavior - `book/src/commands/check.md` — add auto-update section, document `--no-update` - `book/src/commands/update.md` — remove auto-build section and `--build` flag - `book/src/commands/build.md` — add auto-update section, document `--no-update` -- `book/src/commands/search.md` — add auto-update/auto-build section, document `--no-update`/`--no-build`, document first-time search behavior -- `book/src/getting-started.md` — update workflow examples (instant search vs customized build) -- `book/src/recipes/ci.md` — emphasize `--no-update` and `[check].auto_update = false` for deterministic CI -- `book/src/configuration.md` — document new `[check]` and `[build]` sections, update `[search]` section +- `book/src/commands/search.md` — add auto-update/auto-build section, document + `--no-update`/`--no-build`, document first-time search behavior +- `book/src/getting-started.md` — update workflow examples (instant search vs + customized build) +- `book/src/recipes/ci.md` — emphasize `--no-update` and + `[check].auto_update = false` for deterministic CI +- `book/src/configuration.md` — document new `[check]` and `[build]` sections, + update `[search]` section - `docs/spec/commands/*.md` — update command specs to match diff --git a/docs/spec/todos/TODO-0100.md b/docs/spec/todos/TODO-0100.md index 1b4a33e..20f582a 100644 --- a/docs/spec/todos/TODO-0100.md +++ b/docs/spec/todos/TODO-0100.md @@ -13,23 +13,31 @@ blocks: [] ## Summary -The text output tables across all commands are inconsistent. Tables lack headers, have empty columns, and each command's layout was implemented independently. Redesign the text output format for every command with a coherent, uniform style. +The text output tables across all commands are inconsistent. Tables lack +headers, have empty columns, and each command's layout was implemented +independently. Redesign the text output format for every command with a +coherent, uniform style. ## Current state (post TODO-0119/0137) The output architecture is now clean: + - `CommandResult` with flat `Vec` + `Result` -- Each outcome struct implements `Render` → `Vec` (Block::Line, Block::Table, Block::Section) -- `format_text()` in `render.rs` consumes blocks and produces terminal output via `tabled` +- Each outcome struct implements `Render` → `Vec` (Block::Line, + Block::Table, Block::Section) +- `format_text()` in `render.rs` consumes blocks and produces terminal output + via `tabled` - Compact = command outcome only; verbose = step lines + command outcome - Errors force verbose **What's settled:** + - Compact/verbose dispatch model (no more CompactOutcome) - Step lines show timing: `"Read config: example_kb/mdvs.toml (5ms)"` - JSON output is clean (`#[serde(untagged)]`, flat steps array) **What's still wrong — the Render impls themselves:** + - No table headers — columns are unlabeled - Empty columns (e.g., hints column in init when no special characters) - Inconsistent table structure across commands @@ -40,26 +48,28 @@ The output architecture is now clean: All 7 commands' Render impls need redesigning: -| Command | Current output | Issues | -|---------|---------------|--------| -| `init` | Per-field record tables with detail rows | Too verbose for compact, detail rows hard to scan | -| `check` | Violation record tables + new field tables | No headers, kind column is raw enum name | -| `update` | Added/changed/removed record tables | Change tables have 4 columns, hard to read | -| `build` | Embedded/unchanged/removed tables | Detail rows list every file — noisy | -| `search` | Per-hit record tables with chunk text | Good structure, but no headers | -| `info` | Index status table + per-field record tables | Two different table styles mixed | -| `clean` | Two plain lines | Fine as-is | +| Command | Current output | Issues | +| -------- | -------------------------------------------- | ------------------------------------------------- | +| `init` | Per-field record tables with detail rows | Too verbose for compact, detail rows hard to scan | +| `check` | Violation record tables + new field tables | No headers, kind column is raw enum name | +| `update` | Added/changed/removed record tables | Change tables have 4 columns, hard to read | +| `build` | Embedded/unchanged/removed tables | Detail rows list every file — noisy | +| `search` | Per-hit record tables with chunk text | Good structure, but no headers | +| `info` | Index status table + per-field record tables | Two different table styles mixed | +| `clean` | Two plain lines | Fine as-is | ## Approach -1. Define a standard output style guide: header conventions, column alignment, empty column handling, summary line format +1. Define a standard output style guide: header conventions, column alignment, + empty column handling, summary line format 2. Design each command's output on paper first 3. Implement across all commands in a single pass 4. Update book pages to match ## Column width strategy -Use percentage-based column widths relative to terminal width via `tabled`'s `Width::list()`: +Use percentage-based column widths relative to terminal width via `tabled`'s +`Width::list()`: ```rust let total = terminal_size(); @@ -67,12 +77,14 @@ let widths = [total * 20 / 100, total * 50 / 100, total * 30 / 100]; table.with(Width::list(widths)); ``` -Add a helper like `width_percent(total, &[20, 50, 30])` in render.rs so all commands use the same mechanism. +Add a helper like `width_percent(total, &[20, 50, 30])` in render.rs so all +commands use the same mechanism. ## Files - `src/outcome/commands/*.rs` — all 7 command Render impls -- `src/outcome/*.rs` — leaf step Render impls (minor — step lines are already fine) +- `src/outcome/*.rs` — leaf step Render impls (minor — step lines are already + fine) - `src/render.rs` — add width helpers, possibly update `format_text()` - `src/block.rs` — possibly extend Block::Table with header support - `book/src/commands/*.md` — update output examples diff --git a/docs/spec/todos/TODO-0101.md b/docs/spec/todos/TODO-0101.md index 16eeb57..ee49c88 100644 --- a/docs/spec/todos/TODO-0101.md +++ b/docs/spec/todos/TODO-0101.md @@ -32,37 +32,64 @@ files_updated: ## Summary -Add a `markdown` output format alongside the existing formats, and rename the current `text` format to `pretty`. The `--output` / `-o` flag becomes `pretty|markdown|json`, with `pretty` as default for TTY and `markdown` as the most useful non-TTY format. +Add a `markdown` output format alongside the existing formats, and rename the +current `text` format to `pretty`. The `--output` / `-o` flag becomes +`pretty|markdown|json`, with `pretty` as default for TTY and `markdown` as the +most useful non-TTY format. -The 2026-06-16 design discussion considered adding a *third* "compact" / "agent" format alongside markdown and explicitly rejected it. Independent benchmarks (see "Design decision" below) consistently rank Markdown above JSON, YAML, CSV, and TOON on LLM retrieval accuracy across both flat and moderately-nested data. Markdown is the agent format. We don't need a separate one. +The 2026-06-16 design discussion considered adding a _third_ "compact" / "agent" +format alongside markdown and explicitly rejected it. Independent benchmarks +(see "Design decision" below) consistently rank Markdown above JSON, YAML, CSV, +and TOON on LLM retrieval accuracy across both flat and moderately-nested data. +Markdown is the agent format. We don't need a separate one. ## Formats (final) -| Format | Tables | Width | Multiline cells | Use case | -|---|---|---|---|---| -| `pretty` | box-drawing (tabled) | terminal width | yes | interactive terminal use | -| `markdown` | pipe-separated | adapts to content | no, single-line | piping, docs, copy-paste, **agent consumption** | -| `json` | — | — | — | programmatic consumers, `jq` pipelines, strict-contract callers | +| Format | Tables | Width | Multiline cells | Use case | +| ---------- | -------------------- | ----------------- | --------------- | --------------------------------------------------------------- | +| `pretty` | box-drawing (tabled) | terminal width | yes | interactive terminal use | +| `markdown` | pipe-separated | adapts to content | no, single-line | piping, docs, copy-paste, **agent consumption** | +| `json` | — | — | — | programmatic consumers, `jq` pipelines, strict-contract callers | ## Design decision: why Markdown is also the agent format -Independent benchmarks on LLM retrieval and parse accuracy (i.e. how well a model uses formatted data, not just how few tokens it costs to transmit) consistently put Markdown / Markdown-KV at or near the top across format families: - -- A 12-format benchmark from improvingagents.com (cited via İsmail Kağan Acar's dev.to writeup) ranked Markdown-KV at 60.7% on tabular retrieval against JSON 52.3%, CSV 44.3%, and **TOON 9th at 47.5%**. On nested data Markdown 54.3%, YAML 62.1%, JSON 50.3%, **TOON last at 43.1%**. -- arXiv:2603.03306 ("Token-Oriented Object Notation vs JSON…", Feb 2026, independent of TOON authors): plain JSON beats TOON on overall accuracy. -- arXiv:2601.12014 ("Are LLMs Ready for TOON?", Jan 2026): TOON loses structural correctness without native model support. -- Floris Fok (Medium): Markdown achieves highest parse accuracy (94%) in one classification benchmark and saves ~16% tokens vs JSON. +Independent benchmarks on LLM retrieval and parse accuracy (i.e. how well a +model uses formatted data, not just how few tokens it costs to transmit) +consistently put Markdown / Markdown-KV at or near the top across format +families: + +- A 12-format benchmark from improvingagents.com (cited via İsmail Kağan Acar's + dev.to writeup) ranked Markdown-KV at 60.7% on tabular retrieval against JSON + 52.3%, CSV 44.3%, and **TOON 9th at 47.5%**. On nested data Markdown 54.3%, + YAML 62.1%, JSON 50.3%, **TOON last at 43.1%**. +- arXiv:2603.03306 ("Token-Oriented Object Notation vs JSON…", Feb 2026, + independent of TOON authors): plain JSON beats TOON on overall accuracy. +- arXiv:2601.12014 ("Are LLMs Ready for TOON?", Jan 2026): TOON loses structural + correctness without native model support. +- Floris Fok (Medium): Markdown achieves highest parse accuracy (94%) in one + classification benchmark and saves ~16% tokens vs JSON. We also explicitly rejected: -- **TOON** — independent benchmarks show it loses on accuracy across both flat and nested data; the "40-50% token savings" claim measures against pretty-printed JSON, not minified JSON; single primary maintainer; no major framework adoption. -- **YAML** — actually 15-25% *larger* than minified JSON despite folklore, plus indentation fragility. -- **CSV / TSV** — underperforms Markdown-KV on retrieval (~44% vs 60%) and can't represent nesting. -- **A separate "compact" / "agent" format** — Markdown already wins on accuracy and is reasonably token-efficient. A second format adds maintenance surface without evidence-backed upside. If high-volume diagnostics ever justify a denser format, **lint-style** (`path:field:kind detail`, the rustc/eslint convention) is the right shape and can be added later — or used inside the Markdown renderer for the `check` outcome's violations list specifically. +- **TOON** — independent benchmarks show it loses on accuracy across both flat + and nested data; the "40-50% token savings" claim measures against + pretty-printed JSON, not minified JSON; single primary maintainer; no major + framework adoption. +- **YAML** — actually 15-25% _larger_ than minified JSON despite folklore, plus + indentation fragility. +- **CSV / TSV** — underperforms Markdown-KV on retrieval (~44% vs 60%) and can't + represent nesting. +- **A separate "compact" / "agent" format** — Markdown already wins on accuracy + and is reasonably token-efficient. A second format adds maintenance surface + without evidence-backed upside. If high-volume diagnostics ever justify a + denser format, **lint-style** (`path:field:kind detail`, the rustc/eslint + convention) is the right shape and can be added later — or used inside the + Markdown renderer for the `check` outcome's violations list specifically. ## Lint-style inside Markdown for violations -The `check` and `build` outcomes' violation lists should render as lint-style lines inside a `## Violations` Markdown section, e.g.: +The `check` and `build` outcomes' violation lists should render as lint-style +lines inside a `## Violations` Markdown section, e.g.: ``` ## Violations @@ -72,18 +99,24 @@ The `check` and `build` outcomes' violation lists should render as lint-style li - `posts/y.md::unrepresentable` — yaml→json failed at line 4 ``` -This captures the training-data-saturated diagnostic shape (which agents parse fluently from `rustc`, `eslint`, `tsc`) inside the standard Markdown format. Empty case must emit something explicit (e.g. `_0 violations across N files_`) — silent empty reads as "did the command get truncated?" to an agent. +This captures the training-data-saturated diagnostic shape (which agents parse +fluently from `rustc`, `eslint`, `tsc`) inside the standard Markdown format. +Empty case must emit something explicit (e.g. `_0 violations across N files_`) — +silent empty reads as "did the command get truncated?" to an agent. ## Default-format selection -Add `default_output_format` to `mdvs.toml` (under a new `[output]` section or top-level — TBD during implementation). Resolution order: +Add `default_output_format` to `mdvs.toml` (under a new `[output]` section or +top-level — TBD during implementation). Resolution order: 1. CLI flag `--output / -o` (highest priority) 2. `mdvs.toml` `default_output_format` if present 3. TTY autodetect: stdout is a terminal → `pretty`; stdout is piped → `markdown` 4. Hard-coded fallback: `pretty` -TTY autodetection uses `std::io::IsTerminal` (stable since 1.70, no extra crate). This matches the standard CLI convention (gh, ripgrep, etc.) — pipe-to-anything gives sensible output. +TTY autodetection uses `std::io::IsTerminal` (stable since 1.70, no extra +crate). This matches the standard CLI convention (gh, ripgrep, etc.) — +pipe-to-anything gives sensible output. ## Renaming (`text` → `pretty`) @@ -91,36 +124,58 @@ TTY autodetection uses `std::io::IsTerminal` (stable since 1.70, no extra crate) - `format_text()` → `format_pretty()` in `render.rs` - The `Render` trait stays as-is — `Block` IR is format-agnostic - Update CLI help, book pages, and any spec references -- **Pre-1.0 break**: `--output text` is dropped, no alias. Bump minor version; mention in CHANGELOG. (Considered keeping `text` as a hidden alias through v0.x; rejected as not worth the complexity for a pre-1.0 rename.) +- **Pre-1.0 break**: `--output text` is dropped, no alias. Bump minor version; + mention in CHANGELOG. (Considered keeping `text` as a hidden alias through + v0.x; rejected as not worth the complexity for a pre-1.0 rename.) ## Implementation shape (post-TODO-0189) -TODO-0189 collapsed the per-command output dispatch into `CommandResult::render(&self, format: &OutputFormat, verbose: bool) -> Result` on `step.rs`. After that refactor, adding `OutputFormat::Markdown` is a two-touch change: +TODO-0189 collapsed the per-command output dispatch into +`CommandResult::render(&self, format: &OutputFormat, verbose: bool) -> Result` +on `step.rs`. After that refactor, adding `OutputFormat::Markdown` is a +two-touch change: 1. Add `OutputFormat::Markdown` variant in `output.rs`. -2. Add the `(OutputFormat::Markdown, _)` arms in `CommandResult::render` calling the existing `format_markdown` in `render.rs` (already implemented as a basic pipe-table formatter; needs polish for narrative framing and section headers). +2. Add the `(OutputFormat::Markdown, _)` arms in `CommandResult::render` calling + the existing `format_markdown` in `render.rs` (already implemented as a basic + pipe-table formatter; needs polish for narrative framing and section + headers). -The rename + default-format-selection touches more (every clap reference, the help text, every doc, the new TTY-autodetect path in `main.rs`) but is structurally simple. +The rename + default-format-selection touches more (every clap reference, the +help text, every doc, the new TTY-autodetect path in `main.rs`) but is +structurally simple. ## Implementation plan (one branch, four commits, one PR) -Single branch `feat/output-markdown` (or similar — name TBD at branch time). Each commit below is self-contained and reviewable on its own; the PR combines them. +Single branch `feat/output-markdown` (or similar — name TBD at branch time). +Each commit below is self-contained and reviewable on its own; the PR combines +them. ### Commit 1 — `feat(output): add Markdown variant` -Wires up the new format. Smallest first commit, validates the two-touch claim from TODO-0189. +Wires up the new format. Smallest first commit, validates the two-touch claim +from TODO-0189. -- Polish `format_markdown` in `render.rs`: section headers via `##` (verify), empty-state lines (`_0 violations across N files_` etc.), multiline-cell handling (collapse to `
` or truncate — Markdown tables can't hold newlines). +- Polish `format_markdown` in `render.rs`: section headers via `##` (verify), + empty-state lines (`_0 violations across N files_` etc.), multiline-cell + handling (collapse to `
` or truncate — Markdown tables can't hold + newlines). - Add `OutputFormat::Markdown` in `output.rs`. - Add `Markdown` arms in `CommandResult::render`. -- Unit tests mirroring the existing `render_*` tests; assert no trailing newline. -- Manual sweep against `example_kb`: `init`, `check`, `build`, `search`, `info`, `update`, `clean`, `export-jsonschema`. +- Unit tests mirroring the existing `render_*` tests; assert no trailing + newline. +- Manual sweep against `example_kb`: `init`, `check`, `build`, `search`, `info`, + `update`, `clean`, `export-jsonschema`. -**Deferred to a follow-up TODO**: lint-style violations inside Markdown (the bullet-list shape sketched earlier in this file). Clean implementation requires either a new `Block` variant or a format-aware `Render` impl — both bigger than v1 needs. v1 ships violations as Markdown tables. +**Deferred to a follow-up TODO**: lint-style violations inside Markdown (the +bullet-list shape sketched earlier in this file). Clean implementation requires +either a new `Block` variant or a format-aware `Render` impl — both bigger than +v1 needs. v1 ships violations as Markdown tables. ### Commit 2 — `refactor(output): rename Text to Pretty` -Mechanical, noisy diff. Should be easy to review since logical behaviour is unchanged. +Mechanical, noisy diff. Should be easy to review since logical behaviour is +unchanged. - `OutputFormat::Text` → `OutputFormat::Pretty` in `output.rs`. - `format_text` → `format_pretty` in `render.rs`. Update `step.rs` import. @@ -131,45 +186,67 @@ Mechanical, noisy diff. Should be easy to review since logical behaviour is unch Config + resolution chain. -- Add `default_output_format` field to `MdvsToml` (top-level, not in a new `[output]` section — single field, no need for a section to grow into). -- Add an invariant in `MdvsToml::validate()` rejecting values outside `pretty|markdown|json`. +- Add `default_output_format` field to `MdvsToml` (top-level, not in a new + `[output]` section — single field, no need for a section to grow into). +- Add an invariant in `MdvsToml::validate()` rejecting values outside + `pretty|markdown|json`. - Resolution chain in `main.rs`: ```rust let format = cli.output .or_else(|| config.default_output_format) .unwrap_or_else(|| if std::io::stdout().is_terminal() { Pretty } else { Markdown }); ``` -- Refactor `--output` to `Option` so "user didn't specify" is distinguishable from "user explicitly chose Pretty". -- Tests: config-set-pretty + isatty → pretty, config-unset + non-tty → markdown, CLI-flag-wins, invalid config rejected. +- Refactor `--output` to `Option` so "user didn't specify" is + distinguishable from "user explicitly chose Pretty". +- Tests: config-set-pretty + isatty → pretty, config-unset + non-tty → markdown, + CLI-flag-wins, invalid config rejected. ### Commit 4 — `docs: update README, SKILL.md, mdbook, specs for new output formats` -Final pass over the docs surface so everything reflects the v0.x state after the rename + new variants + default selection. Don't skip this — every reference to `--output text` becomes wrong after commit 2. +Final pass over the docs surface so everything reflects the v0.x state after the +rename + new variants + default selection. Don't skip this — every reference to +`--output text` becomes wrong after commit 2. Touch points: -- **`README.md`** (top-level) — any `--output` mentions in the usage / examples section. -- **`crates/mdvs/skills/mdvs/SKILL.md`** — the agent skill: update format roster, default-selection behaviour, recommended format for agent use (Markdown). -- **`book/src/commands/*.md`** — every command page's `--output` / `-o` reference. -- **`book/src/configuration.md`** — document `default_output_format` + the TTY autodetect rule + the priority chain. +- **`README.md`** (top-level) — any `--output` mentions in the usage / examples + section. +- **`crates/mdvs/skills/mdvs/SKILL.md`** — the agent skill: update format + roster, default-selection behaviour, recommended format for agent use + (Markdown). +- **`book/src/commands/*.md`** — every command page's `--output` / `-o` + reference. +- **`book/src/configuration.md`** — document `default_output_format` + the TTY + autodetect rule + the priority chain. - **`book/src/SUMMARY.md`** — verify no orphan references. - **`docs/spec/architecture.md`** — output-format section, if any. -- **`docs/spec/search.md`** / **`docs/spec/shared.md`** / **`docs/spec/commands/*.md`** — sweep for `Text` or `text` references. -- **`CHANGELOG.md`** — note the `text` → `pretty` rename + new `markdown` format + new `default_output_format` config field as a single entry under the upcoming version. -- **`example_kb/mdvs.toml`** — add `default_output_format = "markdown"` commented out, demonstrating the field exists. +- **`docs/spec/search.md`** / **`docs/spec/shared.md`** / + **`docs/spec/commands/*.md`** — sweep for `Text` or `text` references. +- **`CHANGELOG.md`** — note the `text` → `pretty` rename + new `markdown` + format + new `default_output_format` config field as a single entry under the + upcoming version. +- **`example_kb/mdvs.toml`** — add `default_output_format = "markdown"` + commented out, demonstrating the field exists. -After this commit, `grep -ri "output text\|OutputFormat::Text\|format_text" .` should return zero hits outside the changelog. +After this commit, `grep -ri "output text\|OutputFormat::Text\|format_text" .` +should return zero hits outside the changelog. ## Files See per-commit file lists above. Net touchpoints: -- `crates/mdvs/src/output.rs`, `render.rs`, `step.rs`, `schema/config.rs`, `main.rs` -- Docs: `README.md`, `book/src/**`, `docs/spec/**`, `crates/mdvs/skills/mdvs/SKILL.md`, `CHANGELOG.md` +- `crates/mdvs/src/output.rs`, `render.rs`, `step.rs`, `schema/config.rs`, + `main.rs` +- Docs: `README.md`, `book/src/**`, `docs/spec/**`, + `crates/mdvs/skills/mdvs/SKILL.md`, `CHANGELOG.md` - Examples: `example_kb/mdvs.toml` ## Out of scope -- A second output format optimized for "agent compactness". Markdown is the agent format. Revisit only if real users report a token problem after we ship Markdown. -- Wire-protocol-style formats (NDJSON, msgpack, protobuf, etc.) — not motivated by any current user. -- Per-command compact renderers (per-Outcome `render_compact_text`). Not needed given the Markdown decision. +- A second output format optimized for "agent compactness". Markdown is the + agent format. Revisit only if real users report a token problem after we ship + Markdown. +- Wire-protocol-style formats (NDJSON, msgpack, protobuf, etc.) — not motivated + by any current user. +- Per-command compact renderers (per-Outcome `render_compact_text`). Not needed + given the Markdown decision. diff --git a/docs/spec/todos/TODO-0102.md b/docs/spec/todos/TODO-0102.md index 30850a0..a770b51 100644 --- a/docs/spec/todos/TODO-0102.md +++ b/docs/spec/todos/TODO-0102.md @@ -12,15 +12,21 @@ blocks: [] ## Summary -Create a SKILL.md following the open Agent Skills standard so that users' AI coding agents know how to use mdvs. Distribute via a `mdvs create-skill` CLI command (auto-detects agent directories) and a GitHub skills repo for `npx skills add` / skills.sh listing. +Create a SKILL.md following the open Agent Skills standard so that users' AI +coding agents know how to use mdvs. Distribute via a `mdvs create-skill` CLI +command (auto-detects agent directories) and a GitHub skills repo for +`npx skills add` / skills.sh listing. ## Details ### 1. The SKILL.md -Standard format per [agentskills.io/specification](https://agentskills.io/specification). YAML frontmatter (`name`, `description`) + markdown body. +Standard format per +[agentskills.io/specification](https://agentskills.io/specification). YAML +frontmatter (`name`, `description`) + markdown body. **Frontmatter:** + ```yaml --- name: mdvs @@ -32,7 +38,8 @@ description: >- --- ``` -**Body content** — directive tone, concise (<500 lines, <5000 tokens recommended): +**Body content** — directive tone, concise (<500 lines, <5000 tokens +recommended): - One-line tool description - Decision tree: user intent → command @@ -45,12 +52,14 @@ description: >- - Clean build artifacts → `mdvs clean ` - Always use `--output json` for structured, parseable output - Exit codes: 0 = success, non-zero = violations/errors -- Key concepts: `mdvs.toml` (committed schema config), `.mdvs/` (gitignored build artifact), validation works without building +- Key concepts: `mdvs.toml` (committed schema config), `.mdvs/` (gitignored + build artifact), validation works without building - Common patterns (e.g. "after creating a markdown file, run `mdvs check`") ### 2. `mdvs create-skill` command -New CLI command that writes the SKILL.md into the user's project. Non-interactive. +New CLI command that writes the SKILL.md into the user's project. +Non-interactive. **Behavior:** @@ -60,22 +69,28 @@ New CLI command that writes the SKILL.md into the user's project. Non-interactiv - `.cursor/skills/` - `.windsurf/skills/` - `.gemini/skills/` -2. Install `mdvs.md` (containing the SKILL.md content) into **all** detected directories +2. Install `mdvs.md` (containing the SKILL.md content) into **all** detected + directories 3. If no agent directories found, default to `.claude/skills/` (creating it) 4. Print each path written to 5. Refuse to overwrite existing files unless `--force` is passed **Flags:** -- `--force` — overwrite existing skill files (useful when upgrading to a new skill version) + +- `--force` — overwrite existing skill files (useful when upgrading to a new + skill version) - `--path ` — override auto-detection, install into a specific directory -**Output:** list of paths written, consistent with other mdvs commands. Supports `--output json`. +**Output:** list of paths written, consistent with other mdvs commands. Supports +`--output json`. ### 3. GitHub skills repo for skills.sh -Separate public repo (e.g. `mdvs/mdvs-skill` or `edoardo-chio/mdvs-skill`) for distribution via `npx skills add`. +Separate public repo (e.g. `mdvs/mdvs-skill` or `edoardo-chio/mdvs-skill`) for +distribution via `npx skills add`. **Repo structure:** + ``` mdvs-skill/ ├── mdvs/ @@ -85,12 +100,16 @@ mdvs-skill/ ``` **How skills.sh listing works:** + - No submission process — fully automatic -- When anyone runs `npx skills add /mdvs-skill`, anonymous telemetry records it -- The skill auto-appears on [skills.sh](https://skills.sh) and ranks by install count +- When anyone runs `npx skills add /mdvs-skill`, anonymous telemetry + records it +- The skill auto-appears on [skills.sh](https://skills.sh) and ranks by install + count - Users can also discover it via `npx skills find mdvs` **README should include:** + ``` ## Install npx skills add /mdvs-skill @@ -98,11 +117,16 @@ npx skills add /mdvs-skill ### Naming constraints (from spec) -- `name` field: lowercase `a-z`, numbers, hyphens only. Pattern: `^[a-z0-9]+(-[a-z0-9]+)*$` +- `name` field: lowercase `a-z`, numbers, hyphens only. Pattern: + `^[a-z0-9]+(-[a-z0-9]+)*$` - Must match parent directory name exactly - Max 64 characters - `mdvs` fits perfectly ### What's novel -No CLI tool currently ships a `create-skill` subcommand — every other project (Supabase, Prisma, Stripe, etc.) delegates to `npx skills add` from an external repo. The `mdvs create-skill` command would be a first: zero-dependency, self-contained skill installation. Both distribution paths (CLI command + skills repo) serve the same SKILL.md content. +No CLI tool currently ships a `create-skill` subcommand — every other project +(Supabase, Prisma, Stripe, etc.) delegates to `npx skills add` from an external +repo. The `mdvs create-skill` command would be a first: zero-dependency, +self-contained skill installation. Both distribution paths (CLI command + skills +repo) serve the same SKILL.md content. diff --git a/docs/spec/todos/TODO-0103.md b/docs/spec/todos/TODO-0103.md index e1c42cf..d2f4eb8 100644 --- a/docs/spec/todos/TODO-0103.md +++ b/docs/spec/todos/TODO-0103.md @@ -14,34 +14,51 @@ blocks: [] ## Summary -Three invariants in `mdvs.toml` are assumed but not enforced. Hand-edited configs can silently violate them. Add validation that catches all three on config load. +Three invariants in `mdvs.toml` are assumed but not enforced. Hand-edited +configs can silently violate them. Add validation that catches all three on +config load. ## Details ### Invariant 1: ignore and [[fields.field]] are mutually exclusive -A field name cannot appear in both the `[fields].ignore` list and as a `[[fields.field]]` entry. Currently, if a field is in both, the ignore check silently wins during validation and the `[[fields.field]]` entry is dead config. Should be a hard error. +A field name cannot appear in both the `[fields].ignore` list and as a +`[[fields.field]]` entry. Currently, if a field is in both, the ignore check +silently wins during validation and the `[[fields.field]]` entry is dead config. +Should be a hard error. ### Invariant 2: all globs must have valid format -All glob patterns in `allowed` and `required` must end with `/*` or `/**`, or be exactly `*` or `**`. The inference algorithm only produces these four formats. Bare file paths (e.g., `blog/post.md`) or arbitrary patterns (e.g., `blog/*.md`) are not valid — they don't match the directory-level semantics of allowed/required. +All glob patterns in `allowed` and `required` must end with `/*` or `/**`, or be +exactly `*` or `**`. The inference algorithm only produces these four formats. +Bare file paths (e.g., `blog/post.md`) or arbitrary patterns (e.g., `blog/*.md`) +are not valid — they don't match the directory-level semantics of +allowed/required. -Valid: `**`, `*`, `blog/**`, `people/*`, `projects/alpha/notes/**` -Invalid: `blog/post.md`, `blog`, `*.md`, `blog/drafts` +Valid: `**`, `*`, `blog/**`, `people/*`, `projects/alpha/notes/**` Invalid: +`blog/post.md`, `blog`, `*.md`, `blog/drafts` ### Invariant 3: required ⊆ allowed -Every glob in a field's `required` list must be covered by some glob in its `allowed` list. The inference algorithm guarantees this by construction, but manual edits can break it. +Every glob in a field's `required` list must be covered by some glob in its +`allowed` list. The inference algorithm guarantees this by construction, but +manual edits can break it. -Coverage is checked via `globset`: strip the `/*` or `/**` suffix from the required pattern to get the directory path, then test if that path matches any allowed glob. This handles cases like `allowed = ["meetings/**"]` covering `required = ["meetings/all-hands/**"]` correctly. +Coverage is checked via `globset`: strip the `/*` or `/**` suffix from the +required pattern to get the directory path, then test if that path matches any +allowed glob. This handles cases like `allowed = ["meetings/**"]` covering +`required = ["meetings/all-hands/**"]` correctly. Special cases: + - `**` in allowed covers any required glob - `*` in allowed covers `*` in required (root-level files) ### Where to add -Add `MdvsToml::validate()` method in `src/schema/config.rs`. Call it from `run_read_config()` in `src/pipeline/read_config.rs` right after successful parsing. This way every command that loads config gets validation for free. +Add `MdvsToml::validate()` method in `src/schema/config.rs`. Call it from +`run_read_config()` in `src/pipeline/read_config.rs` right after successful +parsing. This way every command that loads config gets validation for free. ### Expected behavior @@ -52,5 +69,8 @@ Add `MdvsToml::validate()` method in `src/schema/config.rs`. Call it from `run_r ### Documentation After implementation, update the mdBook: -- `book/src/configuration.md` — document the three invariants in the `[fields]` section -- `book/src/concepts/schema.md` — the `required ⊆ allowed` invariant is already mentioned, add the other two + +- `book/src/configuration.md` — document the three invariants in the `[fields]` + section +- `book/src/concepts/schema.md` — the `required ⊆ allowed` invariant is already + mentioned, add the other two diff --git a/docs/spec/todos/TODO-0104.md b/docs/spec/todos/TODO-0104.md index 45bfb88..152d623 100644 --- a/docs/spec/todos/TODO-0104.md +++ b/docs/spec/todos/TODO-0104.md @@ -13,33 +13,45 @@ blocks: [] ## Summary -Internal columns are currently prefixed in parquet storage (e.g., `_file_id`, `_filename`). This is unnecessary — the prefix should be a view-level concern applied only at search time. This redesign removes the prefix from storage, renames `filename` to `filepath`, drops the `[storage]` section, and gives users full control over internal column aliases via `[search]`. +Internal columns are currently prefixed in parquet storage (e.g., `_file_id`, +`_filename`). This is unnecessary — the prefix should be a view-level concern +applied only at search time. This redesign removes the prefix from storage, +renames `filename` to `filepath`, drops the `[storage]` section, and gives users +full control over internal column aliases via `[search]`. ## Current State -- Internal columns stored with configurable prefix (default `_`): `_file_id`, `_filename`, `_data`, `_content_hash`, `_built_at` -- `[storage].internal_prefix` controls the prefix — changing it requires `--force` rebuild -- `check_reserved_names()` at init/update blocks frontmatter fields colliding with prefixed names -- Users must know about the `_` prefix to filter by path: `--where "_filename LIKE 'blog/%'"` +- Internal columns stored with configurable prefix (default `_`): `_file_id`, + `_filename`, `_data`, `_content_hash`, `_built_at` +- `[storage].internal_prefix` controls the prefix — changing it requires + `--force` rebuild +- `check_reserved_names()` at init/update blocks frontmatter fields colliding + with prefixed names +- Users must know about the `_` prefix to filter by path: + `--where "_filename LIKE 'blog/%'"` - `[storage]` section is hidden and rarely used ## Proposed Design ### Storage (parquet) — fixed names, no prefix -Internal columns get fixed, unprefixed names. Rename `filename` to `filepath` (more descriptive, less collision-prone): +Internal columns get fixed, unprefixed names. Rename `filename` to `filepath` +(more descriptive, less collision-prone): -| Column | Old name | New name | -|---|---|---| -| File ID | `_file_id` | `file_id` | -| File path | `_filename` | `filepath` | -| Frontmatter | `_data` | `data` | -| Content hash | `_content_hash` | `content_hash` | -| Build timestamp | `_built_at` | `built_at` | +| Column | Old name | New name | +| --------------- | --------------- | -------------- | +| File ID | `_file_id` | `file_id` | +| File path | `_filename` | `filepath` | +| Frontmatter | `_data` | `data` | +| Content hash | `_content_hash` | `content_hash` | +| Build timestamp | `_built_at` | `built_at` | -Chunks parquet similarly: `_chunk_id` → `chunk_id`, `_file_id` → `file_id`, `_chunk_index` → `chunk_index`, `_start_line` → `start_line`, `_end_line` → `end_line`, `_embedding` → `embedding`. +Chunks parquet similarly: `_chunk_id` → `chunk_id`, `_file_id` → `file_id`, +`_chunk_index` → `chunk_index`, `_start_line` → `start_line`, `_end_line` → +`end_line`, `_embedding` → `embedding`. -Storage names are fixed and not configurable. This is an internal implementation detail. +Storage names are fixed and not configurable. This is an internal implementation +detail. ### View (search time) — aliases and prefix applied here @@ -55,7 +67,8 @@ CREATE VIEW files_v AS SELECT FROM files ``` -**No alias/prefix by default.** Internal columns are exposed by their raw names: `filepath`, `file_id`, `content_hash`, `built_at`. Users query with bare names: +**No alias/prefix by default.** Internal columns are exposed by their raw names: +`filepath`, `file_id`, `content_hash`, `built_at`. Users query with bare names: ```bash --where "filepath LIKE 'blog/%'" @@ -63,7 +76,9 @@ FROM files ### Collision handling — search time only -When building the `files_v` view, check if any frontmatter field name collides with an internal column name. If so, **hard error at search time** with a clear message: +When building the `files_v` view, check if any frontmatter field name collides +with an internal column name. If so, **hard error at search time** with a clear +message: ``` field 'filepath' collides with internal column 'filepath' — resolve by: @@ -72,7 +87,9 @@ field 'filepath' collides with internal column 'filepath' — resolve by: - renaming the frontmatter field ``` -**No collision check at init/update/build.** Those commands don't create the view — collisions only matter for `--where` in search. `check_reserved_names()` is removed entirely (or becomes a non-blocking warning in init output). +**No collision check at init/update/build.** Those commands don't create the +view — collisions only matter for `--where` in search. `check_reserved_names()` +is removed entirely (or becomes a non-blocking warning in init output). ### Config changes @@ -92,6 +109,7 @@ internal_prefix = "" # default: no prefix. Set "_" to get _filepath, _f ``` **Resolution precedence for each internal column:** + 1. If alias exists in `[search.aliases]` → use alias 2. Else if `internal_prefix` is set → use `{prefix}{name}` 3. Else → use raw name @@ -99,31 +117,37 @@ internal_prefix = "" # default: no prefix. Set "_" to get _filepath, _f ### User experience examples **Default (no config needed):** + ```bash mdvs search "query" --where "filepath LIKE 'blog/%'" mdvs search "query" --where "file_id = 'abc-123'" ``` **User has a frontmatter field called `filepath`:** + ``` Error: field 'filepath' collides with internal column — set [search].internal_prefix or add [search.aliases].filepath ``` Resolved with alias: + ```toml [search.aliases] filepath = "path" ``` + ```bash mdvs search "query" --where "path LIKE 'blog/%'" # internal mdvs search "query" --where "filepath = 'some_value'" # frontmatter ``` Or resolved with prefix: + ```toml [search] internal_prefix = "_" ``` + ```bash mdvs search "query" --where "_filepath LIKE 'blog/%'" # internal mdvs search "query" --where "filepath = 'some_value'" # frontmatter @@ -133,11 +157,17 @@ mdvs search "query" --where "filepath = 'some_value'" # frontmatter ### Code changes -- `src/index/storage.rs` — remove `col()` prefix helper, rename `RESERVED_BASE_NAMES`, update all column name constants to unprefixed names. Remove `check_reserved_names()`. -- `src/index/backend.rs` — update all parquet column references to unprefixed names -- `src/search.rs` — `SearchContext::new()` view creation: apply aliases/prefix, detect collisions with frontmatter fields, error with resolution guidance -- `src/schema/config.rs` — remove `StorageConfig` and `[storage]` section. Add `internal_prefix` and `aliases` to `SearchConfig`. -- `src/cmd/build.rs` — remove internal prefix from `BuildMetadata` and parquet metadata. Remove `detect_config_changes` for prefix. +- `src/index/storage.rs` — remove `col()` prefix helper, rename + `RESERVED_BASE_NAMES`, update all column name constants to unprefixed names. + Remove `check_reserved_names()`. +- `src/index/backend.rs` — update all parquet column references to unprefixed + names +- `src/search.rs` — `SearchContext::new()` view creation: apply aliases/prefix, + detect collisions with frontmatter fields, error with resolution guidance +- `src/schema/config.rs` — remove `StorageConfig` and `[storage]` section. Add + `internal_prefix` and `aliases` to `SearchConfig`. +- `src/cmd/build.rs` — remove internal prefix from `BuildMetadata` and parquet + metadata. Remove `detect_config_changes` for prefix. - `src/cmd/init.rs` — remove `check_reserved_names()` call from write_config - `src/cmd/update.rs` — remove `check_reserved_names()` call - `src/pipeline/write_config.rs` — remove `check_reserved_names()` call @@ -146,19 +176,25 @@ mdvs search "query" --where "filepath = 'some_value'" # frontmatter ### Config migration - Existing `[storage].internal_prefix` → move to `[search].internal_prefix` -- Existing parquets have prefixed columns → requires `clean` + rebuild (one-time migration) +- Existing parquets have prefixed columns → requires `clean` + rebuild (one-time + migration) - New `init` writes no `[storage]` section ### Documentation -- `book/src/configuration.md` — remove `[storage]` section, add `internal_prefix` and `aliases` to `[search]` -- `book/src/search-guide.md` — update "Filtering by file path" section (use `filepath` instead of `_filename`) -- `book/src/commands/build.md` — remove references to prefix in config change detection +- `book/src/configuration.md` — remove `[storage]` section, add + `internal_prefix` and `aliases` to `[search]` +- `book/src/search-guide.md` — update "Filtering by file path" section (use + `filepath` instead of `_filename`) +- `book/src/commands/build.md` — remove references to prefix in config change + detection - `book/src/concepts/search.md` — update any references to prefixed column names ### Breaking changes -- Parquet column names change → existing `.mdvs/` must be rebuilt (`clean` + `build`) +- Parquet column names change → existing `.mdvs/` must be rebuilt (`clean` + + `build`) - `[storage]` section no longer recognized → warning or error on load - `_filename` in `--where` no longer works → use `filepath` (or aliased name) -- `check_reserved_names()` removed → frontmatter field `_filename` is now allowed (collision handled at search time instead) +- `check_reserved_names()` removed → frontmatter field `_filename` is now + allowed (collision handled at search time instead) diff --git a/docs/spec/todos/TODO-0105.md b/docs/spec/todos/TODO-0105.md index f385ff2..720de94 100644 --- a/docs/spec/todos/TODO-0105.md +++ b/docs/spec/todos/TODO-0105.md @@ -12,15 +12,21 @@ blocks: [] ## Summary -Write the `recipes/ci.md` book page with a GitHub Actions workflow for running `mdvs check` as a CI linter on PRs. Must be tested in an actual CI environment first. +Write the `recipes/ci.md` book page with a GitHub Actions workflow for running +`mdvs check` as a CI linter on PRs. Must be tested in an actual CI environment +first. ## Details ### What to test -1. **Installation method**: `cargo install mdvs` vs cargo-dist prebuilt binaries. Prebuilt is faster (no compilation), but `cargo install` is simpler to write. -2. **Exit code propagation**: `mdvs check` exits 1 on violations, 2 on errors. Verify CI runners fail the step correctly. -3. **No model needed**: `mdvs check` doesn't require embedding model download. Verify it works without `.mdvs/` directory. +1. **Installation method**: `cargo install mdvs` vs cargo-dist prebuilt + binaries. Prebuilt is faster (no compilation), but `cargo install` is simpler + to write. +2. **Exit code propagation**: `mdvs check` exits 1 on violations, 2 on errors. + Verify CI runners fail the step correctly. +3. **No model needed**: `mdvs check` doesn't require embedding model download. + Verify it works without `.mdvs/` directory. 4. **Runner OS**: test on ubuntu-latest at minimum. ### Workflow sketch @@ -40,6 +46,7 @@ jobs: ### Page content Once tested, write `book/src/recipes/ci.md` covering: + - GitHub Actions workflow (copy-pasteable) - Installation options (cargo install vs prebuilt) - Using JSON output for CI-friendly reporting (`-o json`) diff --git a/docs/spec/todos/TODO-0106.md b/docs/spec/todos/TODO-0106.md index 1e19fd0..013272f 100644 --- a/docs/spec/todos/TODO-0106.md +++ b/docs/spec/todos/TODO-0106.md @@ -18,15 +18,35 @@ related: ## Summary -Extract the links a markdown corpus already contains — markdown `[text](path.md#slug)`, Obsidian wikilinks `[[page]]` / `[[page#Heading]]`, and frontmatter references (`parent = [[index]]` via `is_reference = true`) — into a queryable relationship layer, and fold it into the commands mdvs already has: `search` (relationship filters), `check` (broken / ambiguous links), and `info` (stats). No new top-level command, no separate dataset. - -**Nodes are files and sections** (a per-document heading hierarchy). **Edges are Referential** (the extracted authored links) **and Structural** (the heading tree — parent / child / sibling). **Semantic similarity is not part of the graph** — it is discovery-only and lives in [TODO-0171](TODO-0171.md). Explicit links are stored as a column on the chunk row; petgraph handles traversal at query time. +Extract the links a markdown corpus already contains — markdown +`[text](path.md#slug)`, Obsidian wikilinks `[[page]]` / `[[page#Heading]]`, and +frontmatter references (`parent = [[index]]` via `is_reference = true`) — into a +queryable relationship layer, and fold it into the commands mdvs already has: +`search` (relationship filters), `check` (broken / ambiguous links), and `info` +(stats). No new top-level command, no separate dataset. + +**Nodes are files and sections** (a per-document heading hierarchy). **Edges are +Referential** (the extracted authored links) **and Structural** (the heading +tree — parent / child / sibling). **Semantic similarity is not part of the +graph** — it is discovery-only and lives in [TODO-0171](TODO-0171.md). Explicit +links are stored as a column on the chunk row; petgraph handles traversal at +query time. ## Scope — extracted links only -This TODO covers links **extracted from markdown content** — connections *derivable from the files themselves* (a wikilink, a markdown link, an `is_reference` frontmatter value, the heading structure). They are build state: recreatable, gitignored, never a source of truth. +This TODO covers links **extracted from markdown content** — connections +_derivable from the files themselves_ (a wikilink, a markdown link, an +`is_reference` frontmatter value, the heading structure). They are build state: +recreatable, gitignored, never a source of truth. -**Out of scope:** (1) *declared*, out-of-band references — e.g. a link a user or agent authors between a doc and a specific code symbol, maintained in a committed sidecar with drift / staleness detection over code; that is a separate concern (a distinct traceability project), not mdvs. (2) *semantic* edges — a cosine score is a hypothesis, not a connection; semantic similarity is discovery, handled by TODO-0171. The dividing line for edges: **extracted-and-explicit belongs here; declared-code and inferred-semantic do not.** +**Out of scope:** (1) _declared_, out-of-band references — e.g. a link a user or +agent authors between a doc and a specific code symbol, maintained in a +committed sidecar with drift / staleness detection over code; that is a separate +concern (a distinct traceability project), not mdvs. (2) _semantic_ edges — a +cosine score is a hypothesis, not a connection; semantic similarity is +discovery, handled by TODO-0171. The dividing line for edges: +**extracted-and-explicit belongs here; declared-code and inferred-semantic do +not.** ## Nodes — files and sections (internal hierarchy) @@ -35,61 +55,125 @@ Nodes are at two granularities: - **File** — the document (root node). - **Section** — a heading and its content, identified by `(file, heading-path)`. -A markdown document has an implicit **heading tree**: H1 ⊃ H2 ⊃ H3; headings at the same level under the same parent are siblings; a paragraph belongs to its nearest preceding heading; the file is the root. Markdown nesting is positional, so the tree is reconstructed with a stack pass over the headings (via `tree-sitter-markdown` or a plain heading scan) — deterministic, model-free, cheap, so it is **derived on demand, never stored**. +A markdown document has an implicit **heading tree**: H1 ⊃ H2 ⊃ H3; headings at +the same level under the same parent are siblings; a paragraph belongs to its +nearest preceding heading; the file is the root. Markdown nesting is positional, +so the tree is reconstructed with a stack pass over the headings (via +`tree-sitter-markdown` or a plain heading scan) — deterministic, model-free, +cheap, so it is **derived on demand, never stored**. -The heading tree produces **Structural edges** — `contains` / `parent-of` / `sibling` — a distinct edge origin from Referential links. Keeping them distinct matters: a `--backlinks-of` query must not accidentally return the section next door. Structural edges are derived at query / validate time, never persisted. +The heading tree produces **Structural edges** — `contains` / `parent-of` / +`sibling` — a distinct edge origin from Referential links. Keeping them distinct +matters: a `--backlinks-of` query must not accidentally return the section next +door. Structural edges are derived at query / validate time, never persisted. -**Sections only for v0.** No block / paragraph (`^blockid`) nodes — that finer tier is deferred (and is Obsidian-only anyway, see notations below). +**Sections only for v0.** No block / paragraph (`^blockid`) nodes — that finer +tier is deferred (and is Obsidian-only anyway, see notations below). -**Validation win:** with the section tree, `check` detects **broken heading anchors** — `[[doc#Ghost]]` where the doc exists but the heading doesn't — a sharper integrity signal than "file missing." +**Validation win:** with the section tree, `check` detects **broken heading +anchors** — `[[doc#Ghost]]` where the doc exists but the heading doesn't — a +sharper integrity signal than "file missing." ## Link addressing, types, and notations -A link is `(source-node, target-node)`, each file-or-section. The two granularities fall out of two independent axes: +A link is `(source-node, target-node)`, each file-or-section. The two +granularities fall out of two independent axes: -- **Source granularity — from *where the link is authored*:** frontmatter → *file* source (no body position → the file root); body → *section* source (the section enclosing the link, derived from the link's position + the heading tree). Because links are stored per-chunk, the source section is **derivable**, not separately stored. -- **Target granularity — from *the target syntax*:** no anchor → the file root; a `#heading` / `#slug` anchor → a section. +- **Source granularity — from _where the link is authored_:** frontmatter → + _file_ source (no body position → the file root); body → _section_ source (the + section enclosing the link, derived from the link's position + the heading + tree). Because links are stored per-chunk, the source section is + **derivable**, not separately stored. +- **Target granularity — from _the target syntax_:** no anchor → the file root; + a `#heading` / `#slug` anchor → a section. The four link types are one uniform model: -| | → file | → section | -|---|---|---| +| | → file | → section | +| ---------------------------------- | -------------------- | ----------------------- | | **file** source (frontmatter link) | `parent = [[index]]` | `see = [[guide#Setup]]` | -| **section** source (body link) | body `[[guide]]` | body `[[guide#Setup]]` | +| **section** source (body link) | body `[[guide]]` | body `[[guide#Setup]]` | -**Both notations resolve to the same section node.** Each heading node carries two keys — its raw **text** and its canonical **slug** — so a heading link resolves identically whether written: +**Both notations resolve to the same section node.** Each heading node carries +two keys — its raw **text** and its canonical **slug** — so a heading link +resolves identically whether written: -- Obsidian — `[[file#Heading]]`, nested `[[file#H1#H2]]` (a *path* of heading texts), disambiguated by hierarchy; or -- GitHub / standard markdown — `[text](file.md#slug)` (a *flat slug*), disambiguated by order (`slug`, `slug-1`, `slug-2`). +- Obsidian — `[[file#Heading]]`, nested `[[file#H1#H2]]` (a _path_ of heading + texts), disambiguated by hierarchy; or +- GitHub / standard markdown — `[text](file.md#slug)` (a _flat slug_), + disambiguated by order (`slug`, `slug-1`, `slug-2`). -Pick **one canonical slugger** (github-slugger is the de facto standard) and document it — heading→slug differs slightly across renderers on unicode / punctuation. Because blocks are out of scope, the only Obsidian-exclusive capability (block refs) is gone, so within section-only the two notations reach **full parity**. +Pick **one canonical slugger** (github-slugger is the de facto standard) and +document it — heading→slug differs slightly across renderers on unicode / +punctuation. Because blocks are out of scope, the only Obsidian-exclusive +capability (block refs) is gone, so within section-only the two notations reach +**full parity**. ## Storage — a `links` column on the chunk row (no `edges.lance`) -A separate persisted edge dataset was considered and dropped. Persisting `.mdvs/` state is justified by *recompute cost* (embeddings need model inference); link extraction is a cheap scan — cheaper than the validation pass `check` already runs. A second Lance dataset (its own manifest, fragments, incremental-write story to keep in lockstep with `index.lance`) is a sledgehammer for a list of edges. - -Instead: store the outbound links extracted from each chunk as a **column on that chunk's row in `index.lance`**. - -- Proposed shape: `links: List`. `target_file_id` = `None` when unresolved (keep `raw` so `info` can list broken links); `target_heading` = the anchor as authored (heading-path for Obsidian, slug for GitHub) or `None` for a whole-file link; `notation` tells resolution which key to match. This folds the would-be `broken` / `ambiguous` / `anchor` / `heading` / `source_chunk_id` columns into one nested column. Lance/Arrow handles `List` (same machinery as the existing nested `data` frontmatter struct). -- **Per-chunk granularity is correct**: links physically live in a chunk, so storing them there is faithful and non-redundant. File→file / section edges come from a **group-by `file_id`** during the petgraph projection — part of the O(corpus) projection that already feeds petgraph, so it costs nothing extra. The source section is recovered from the chunk's line-range + heading tree. -- **Frontmatter `is_reference` links are nearly free**: those fields are already stored per-file in the `data` struct. Don't duplicate them into the `links` column — the projection parses the wikilink out of `data.` on read (file-source by definition). Only *body* links need the new column. -- Rides TODO-0173's incremental writes for free: the column is part of the chunk row, so rewriting a file's chunks rewrites its links in the same operation — consistency is automatic, no second incremental path. - -**The one tradeoff, stated honestly:** a column on `index.lance` only exists after `mdvs build`, so the *persisted, queryable* graph couples to the search / build layer. The two-layer invariant is preserved by splitting responsibilities: `check` computes broken / ambiguous links (including broken heading anchors) **on the fly** during its existing scan (model-free, no build required → validation stands alone); `mdvs build` persists the `links` column; the query axes read it. +A separate persisted edge dataset was considered and dropped. Persisting +`.mdvs/` state is justified by _recompute cost_ (embeddings need model +inference); link extraction is a cheap scan — cheaper than the validation pass +`check` already runs. A second Lance dataset (its own manifest, fragments, +incremental-write story to keep in lockstep with `index.lance`) is a +sledgehammer for a list of edges. + +Instead: store the outbound links extracted from each chunk as a **column on +that chunk's row in `index.lance`**. + +- Proposed shape: + `links: List`. + `target_file_id` = `None` when unresolved (keep `raw` so `info` can list + broken links); `target_heading` = the anchor as authored (heading-path for + Obsidian, slug for GitHub) or `None` for a whole-file link; `notation` tells + resolution which key to match. This folds the would-be `broken` / `ambiguous` + / `anchor` / `heading` / `source_chunk_id` columns into one nested column. + Lance/Arrow handles `List` (same machinery as the existing nested + `data` frontmatter struct). +- **Per-chunk granularity is correct**: links physically live in a chunk, so + storing them there is faithful and non-redundant. File→file / section edges + come from a **group-by `file_id`** during the petgraph projection — part of + the O(corpus) projection that already feeds petgraph, so it costs nothing + extra. The source section is recovered from the chunk's line-range + heading + tree. +- **Frontmatter `is_reference` links are nearly free**: those fields are already + stored per-file in the `data` struct. Don't duplicate them into the `links` + column — the projection parses the wikilink out of `data.` on read + (file-source by definition). Only _body_ links need the new column. +- Rides TODO-0173's incremental writes for free: the column is part of the chunk + row, so rewriting a file's chunks rewrites its links in the same operation — + consistency is automatic, no second incremental path. + +**The one tradeoff, stated honestly:** a column on `index.lance` only exists +after `mdvs build`, so the _persisted, queryable_ graph couples to the search / +build layer. The two-layer invariant is preserved by splitting responsibilities: +`check` computes broken / ambiguous links (including broken heading anchors) +**on the fly** during its existing scan (model-free, no build required → +validation stands alone); `mdvs build` persists the `links` column; the query +axes read it. ## Traversal — petgraph at query time -Project `(file_id, links)` + group-by + build `petgraph::DiGraph` over the Referential (and, when asked, Structural) edges, per query. Estimated: ~10ms at 1K files, ~100ms at 10K, ~1s at 100K, ~10s at 1M. At the corpora mdvs actually runs (Refractions 549, example_kb 45, K8s 1,669) this is sub-second and moot. The only thing that would bite is composing a graph filter into *every* search (rebuild petgraph per query) — that's the signal to add a deferred disposable cache (content-hash-keyed, rebuilt from the column). On-the-fly is the v0 answer; the cache stays unbuilt until a real corpus crosses ~100K files. +Project `(file_id, links)` + group-by + build `petgraph::DiGraph` over the +Referential (and, when asked, Structural) edges, per query. Estimated: ~10ms at +1K files, ~100ms at 10K, ~1s at 100K, ~10s at 1M. At the corpora mdvs actually +runs (Refractions 549, `example_kb` 45, K8s 1,669) this is sub-second and moot. +The only thing that would bite is composing a graph filter into _every_ search +(rebuild petgraph per query) — that's the signal to add a deferred disposable +cache (content-hash-keyed, rebuilt from the column). On-the-fly is the v0 +answer; the cache stays unbuilt until a real corpus crosses ~100K files. ## UX — folds into `search` / `check` / `info` (no `mdvs graph` command) -The graph is "just another part of the data," so it doesn't get its own verb. A **search is a set of composable narrowing operators over the document set**; each operator is optional, any subset composes: +The graph is "just another part of the data," so it doesn't get its own verb. A +**search is a set of composable narrowing operators over the document set**; +each operator is optional, any subset composes: -| Axis | What it does | Surface | -|---|---|---| -| Relevance | rank by similarity to an anchor | query string · `--related-to ` (semantic *discovery*, see TODO-0171 — **not a graph edge**) | -| Attributes | filter by typed frontmatter | `--where ""` (existing) | -| Relationships | filter by graph connection | `--connected-to --depth N` · `--backlinks-of ` (file- or section-level, e.g. `--backlinks-of guide#Setup`) | +| Axis | What it does | Surface | +| ------------- | ------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| Relevance | rank by similarity to an anchor | query string · `--related-to ` (semantic _discovery_, see TODO-0171 — **not a graph edge**) | +| Attributes | filter by typed frontmatter | `--where ""` (existing) | +| Relationships | filter by graph connection | `--connected-to --depth N` · `--backlinks-of ` (file- or section-level, e.g. `--backlinks-of guide#Setup`) | ``` mdvs search --backlinks-of guide.md#Setup # who links into that section @@ -100,26 +184,48 @@ mdvs search "calibration" \ The would-be `mdvs graph` namespace dissolves entirely: -- **Navigation / filtering** → flags on `search` (above). A REPL "walk" mode is out — it would violate "no interactive prompts until 1.0." Human spatial navigation belongs to a future TUI / GUI, not the CLI; the CLI stays one-shot + `--output json`, the right shape for the agent audience. -- **Integrity** (broken links, broken heading anchors, ambiguous) → *violations*, so they belong in `check`, model-free. +- **Navigation / filtering** → flags on `search` (above). A REPL "walk" mode is + out — it would violate "no interactive prompts until 1.0." Human spatial + navigation belongs to a future TUI / GUI, not the CLI; the CLI stays + one-shot + `--output json`, the right shape for the agent audience. +- **Integrity** (broken links, broken heading anchors, ambiguous) → + _violations_, so they belong in `check`, model-free. - **Stats** (edge counts, most-linked files/sections) → `info`. ## Semantic is not part of the graph -A cosine score identifies a *possible* connection; it is never *a* connection. The graph asserts only **Referential** (explicit authored links) and **Structural** (heading-tree) edges — both deterministic. Semantic similarity lives entirely in **search as discovery**: `search --related-to ` surfaces similar docs to *suggest* links a human or agent then makes explicit. The workflows that turn similarity into explicit links are [TODO-0171](TODO-0171.md). Nothing semantic is stored as an edge, and `--connected-to` never traverses a semantic "edge" — there is none. +A cosine score identifies a _possible_ connection; it is never _a_ connection. +The graph asserts only **Referential** (explicit authored links) and +**Structural** (heading-tree) edges — both deterministic. Semantic similarity +lives entirely in **search as discovery**: `search --related-to ` surfaces +similar docs to _suggest_ links a human or agent then makes explicit. The +workflows that turn similarity into explicit links are +[TODO-0171](TODO-0171.md). Nothing semantic is stored as an edge, and +`--connected-to` never traverses a semantic "edge" — there is none. ## What to extract -Link forms, all resolved to file paths (and optional section anchors) within the scanned root; external `http(s)://` links are ignored. +Link forms, all resolved to file paths (and optional section anchors) within the +scanned root; external `http(s)://` links are ignored. -1. **Markdown links**: `[text](path.md)` → file; `[text](path.md#slug)` → section (GitHub slug). Relation `markdown_link`. -2. **Wikilinks**: `[[page]]` → file; `[[page#Heading]]` / `[[page#H1#H2]]` → section (Obsidian heading text / path); `[[page|alias]]` display text kept in `raw`. Relation `wikilink`. -3. **Frontmatter references**: fields with `is_reference = true` → `fm_reference` edges (file-source), tagged with the field name; target may be file or section. Derived from the stored `data` struct, not re-scanned. +1. **Markdown links**: `[text](path.md)` → file; `[text](path.md#slug)` → + section (GitHub slug). Relation `markdown_link`. +2. **Wikilinks**: `[[page]]` → file; `[[page#Heading]]` / `[[page#H1#H2]]` → + section (Obsidian heading text / path); `[[page|alias]]` display text kept in + `raw`. Relation `wikilink`. +3. **Frontmatter references**: fields with `is_reference = true` → + `fm_reference` edges (file-source), tagged with the field name; target may be + file or section. Derived from the stored `data` struct, not re-scanned. Resolution rules (configurable under `[graph]`): -- **Wikilink resolution.** Default strict: ambiguous file matches become `LinkAmbiguous` violations in `check` (lexicographic-first applied). `resolve = "proximity"` opts into Obsidian-style closest-match. -- **Broken links.** `LinkBroken` in `check` — covers both a missing *file* and a missing *heading anchor* (file resolves, heading not in its tree). Warning by default; `strict = true` makes them violations. The unresolved `raw` is retained in the column so `info` can list them. +- **Wikilink resolution.** Default strict: ambiguous file matches become + `LinkAmbiguous` violations in `check` (lexicographic-first applied). + `resolve = "proximity"` opts into Obsidian-style closest-match. +- **Broken links.** `LinkBroken` in `check` — covers both a missing _file_ and a + missing _heading anchor_ (file resolves, heading not in its tree). Warning by + default; `strict = true` makes them violations. The unresolved `raw` is + retained in the column so `info` can list them. ## Frontmatter reference declaration @@ -137,7 +243,9 @@ type = "Array(String)" is_reference = true # array form: each element parsed as a wikilink ``` -When `is_reference = true`, the field's string value(s) are parsed as wikilinks during the existing frontmatter validation pass — no separate scan, and no separate storage (the edges derive from the `data` struct on read). +When `is_reference = true`, the field's string value(s) are parsed as wikilinks +during the existing frontmatter validation pass — no separate scan, and no +separate storage (the edges derive from the `data` struct on read). ## Config @@ -152,55 +260,140 @@ strict = false # broken links become violations when true (oth ## Prior art — filter vs rank is the real organizing principle -Surveyed how production systems put relevance + attribute + relationship in one query. Two philosophies: **(A) one query representation** with operators for all three — Cypher/GQL, SurrealQL (SQL + graph arrows), SQL/PGQ `GRAPH_TABLE(... MATCH ...)`, Weaviate GraphQL, Vespa YQL (`nearestNeighbor()` + `rank()`); **(B) multiple retrievers + fusion** — GraphRAG merges vector/graph/keyword retrievers with Reciprocal Rank Fusion (`Σ 1/(k+rank_i)`, rank-based so scores needn't share a scale). - -The cross-cutting lesson, more important than syntax: **each layer is either a hard filter (narrows, boolean) or a soft ranker (orders the survivors)** — Vespa's `rank()` ("retrieve by A, score by B"), Neo4j's pre/in/post-filter taxonomy. Attributes = filter; relevance = ranker; relationship = *either* (filter: "≤N hops"; rank: link-proximity boost). **mdvs already embodies this** — `--where` is a filter, query+`--mode` is a ranker, `hybrid` is already RRF fusion. So graph-as-filter via flags is the natural v0; graph-as-*boost* folds into the existing hybrid/RRF machinery later. - -Sources: [Neo4j vector+filter](https://neo4j.com/blog/genai/vector-search-with-filters-in-neo4j-v2026-01-preview/), [SurrealDB KG-RAG](https://surrealdb.com/blog/knowledge-graph-rag-two-query-patterns-for-smarter-ai-agents), [DuckPGQ SQL/PGQ](https://duckpgq.org/documentation/sql_pgq/), [Weaviate hybrid](https://docs.weaviate.io/weaviate/search/hybrid), [Vespa NN search](https://docs.vespa.ai/en/querying/nearest-neighbor-search), [RAG patterns 2026 / RRF](https://ailearningguides.com/rag-production-patterns-2026/). +Surveyed how production systems put relevance + attribute + relationship in one +query. Two philosophies: **(A) one query representation** with operators for all +three — Cypher/GQL, SurrealQL (SQL + graph arrows), SQL/PGQ +`GRAPH_TABLE(... MATCH ...)`, Weaviate GraphQL, Vespa YQL (`nearestNeighbor()` + +`rank()`); **(B) multiple retrievers + fusion** — GraphRAG merges +vector/graph/keyword retrievers with Reciprocal Rank Fusion (`Σ 1/(k+rank_i)`, +rank-based so scores needn't share a scale). + +The cross-cutting lesson, more important than syntax: **each layer is either a +hard filter (narrows, boolean) or a soft ranker (orders the survivors)** — +Vespa's `rank()` ("retrieve by A, score by B"), Neo4j's pre/in/post-filter +taxonomy. Attributes = filter; relevance = ranker; relationship = _either_ +(filter: "≤N hops"; rank: link-proximity boost). **mdvs already embodies this** +— `--where` is a filter, query+`--mode` is a ranker, `hybrid` is already RRF +fusion. So graph-as-filter via flags is the natural v0; graph-as-_boost_ folds +into the existing hybrid/RRF machinery later. + +Sources: +[Neo4j vector+filter](https://neo4j.com/blog/genai/vector-search-with-filters-in-neo4j-v2026-01-preview/), +[SurrealDB KG-RAG](https://surrealdb.com/blog/knowledge-graph-rag-two-query-patterns-for-smarter-ai-agents), +[DuckPGQ SQL/PGQ](https://duckpgq.org/documentation/sql_pgq/), +[Weaviate hybrid](https://docs.weaviate.io/weaviate/search/hybrid), +[Vespa NN search](https://docs.vespa.ai/en/querying/nearest-neighbor-search), +[RAG patterns 2026 / RRF](https://ailearningguides.com/rag-production-patterns-2026/). ## Comparative note — graphify -The closest neighbouring tool, [graphify](https://github.com/safishamsi/graphify), persists an in-memory NetworkX graph to a single `graph.json` (node-link JSON) plus a derived `graph.html` — because that file *is* its database (its nodes are LLM-extracted concepts with nowhere else to live). mdvs's graph derives from data it already holds, so a standalone graph file would be a redundant second store; mdvs keeps a `links` column and treats `graph.json` / `graph.html` as a *derived export* (`mdvs export`, also giving Obsidian interop). Two robustness points worth carrying, which graphify independently reached: derived graph state should refuse to silently shrink, and directional edges should use a directed representation (`petgraph::DiGraph`). And a design contrast that confirms mdvs's choice: graphify's implicit edges are an **LLM judgment** (`semantically_similar_to`), whereas mdvs keeps implicit similarity out of the graph entirely (discovery-only, TODO-0171) — lighter, deterministic, and no LLM in the index path. +The closest neighbouring tool, +[graphify](https://github.com/safishamsi/graphify), persists an in-memory +NetworkX graph to a single `graph.json` (node-link JSON) plus a derived +`graph.html` — because that file _is_ its database (its nodes are LLM-extracted +concepts with nowhere else to live). mdvs's graph derives from data it already +holds, so a standalone graph file would be a redundant second store; mdvs keeps +a `links` column and treats `graph.json` / `graph.html` as a _derived export_ +(`mdvs export`, also giving Obsidian interop). Two robustness points worth +carrying, which graphify independently reached: derived graph state should +refuse to silently shrink, and directional edges should use a directed +representation (`petgraph::DiGraph`). And a design contrast that confirms mdvs's +choice: graphify's implicit edges are an **LLM judgment** +(`semantically_similar_to`), whereas mdvs keeps implicit similarity out of the +graph entirely (discovery-only, TODO-0171) — lighter, deterministic, and no LLM +in the index path. ## Online search — a separate source axis, deferred -"Search online" is another *source* (`--source local|web|both`), not another operator. It changes mdvs's identity (breaks offline / single-binary; needs API keys; web results aren't markdown-with-typed-frontmatter, so `--where` / schema / validation don't apply). Explicitly opt-in, decided as its own thing later; it must not shape the local unification. +"Search online" is another _source_ (`--source local|web|both`), not another +operator. It changes mdvs's identity (breaks offline / single-binary; needs API +keys; web results aren't markdown-with-typed-frontmatter, so `--where` / schema +/ validation don't apply). Explicitly opt-in, decided as its own thing later; it +must not shape the local unification. ## Still open (do not implement until settled) -1. **Representation for v0**: orthogonal flags (zero parser work, no cross-layer `OR`) vs graph predicates embedded in `--where` (full cross-layer boolean, grows the translator). Leaning flags. -2. **Filter vs rank per graph axis**, and whether graph-as-boost (link-proximity / PageRank fused via RRF) is in v0 or later. -3. **Composition order** for `--connected-to` + a relevance query (pre- / in- / post-filter, per Neo4j). -4. **Structural edges in traversal**: does `--connected-to` walk `parent` / `sibling`, or is hierarchy a separate lens (breadcrumb / `--within `)? Leaning **separate** — keep origins distinct so `--backlinks-of` never returns a sibling section. -5. **Final `links` struct shape** — the exact representation of `target_heading` (store raw anchor + `notation`, or normalize to a resolved heading-path?). +1. **Representation for v0**: orthogonal flags (zero parser work, no cross-layer + `OR`) vs graph predicates embedded in `--where` (full cross-layer boolean, + grows the translator). Leaning flags. +2. **Filter vs rank per graph axis**, and whether graph-as-boost (link-proximity + / PageRank fused via RRF) is in v0 or later. +3. **Composition order** for `--connected-to` + a relevance query (pre- / in- / + post-filter, per Neo4j). +4. **Structural edges in traversal**: does `--connected-to` walk `parent` / + `sibling`, or is hierarchy a separate lens (breadcrumb / `--within `)? + Leaning **separate** — keep origins distinct so `--backlinks-of` never + returns a sibling section. +5. **Final `links` struct shape** — the exact representation of `target_heading` + (store raw anchor + `notation`, or normalize to a resolved heading-path?). 6. **Canonical slugger** — confirm github-slugger and document the algorithm. -7. **GUI** (separate, not-yet-written TODO): overview-poster (static HTML, force-directed) vs navigation-cockpit (ratatui TUI). mdvs should offer `mdvs export --format graphjson|html` as a *derived* artifact from the `links` column. +7. **GUI** (separate, not-yet-written TODO): overview-poster (static HTML, + force-directed) vs navigation-cockpit (ratatui TUI). mdvs should offer + `mdvs export --format graphjson|html` as a _derived_ artifact from the + `links` column. ## Scope (v0) — what ships -- Extract markdown links, wikilinks (both with optional heading anchors, both notations), and `is_reference` frontmatter refs during `mdvs build`; persist as the `links` column. -- Build the per-document heading tree (derived) → section nodes + Structural edges. -- `check`: `LinkBroken` (missing file **or** missing heading anchor) and `LinkAmbiguous`, computed on the fly (model-free, no build required). -- `search`: `--connected-to [--depth N]`, `--backlinks-of ` (file- or section-level), composed with the existing modes and `--where`. -- `info`: edge count, broken / ambiguous counts, top-N most-linked files/sections. -- `[graph]` config (`enabled` default false, `resolve`, `strict`) and the `is_reference` schema attribute. -- Multi-hop via petgraph projection, no cache. **Sections only** (blocks deferred). - -Semantic `--related-to` and the discovery workflows are **not here** — they are TODO-0171. Deferred, gated on evidence: a disposable traversal cache (only past ~100K files); graph-as-rank-boost. **`--connected-to --depth > 1` needs a benchmark** on a 50K+ synthetic corpus before promotion. +- Extract markdown links, wikilinks (both with optional heading anchors, both + notations), and `is_reference` frontmatter refs during `mdvs build`; persist + as the `links` column. +- Build the per-document heading tree (derived) → section nodes + Structural + edges. +- `check`: `LinkBroken` (missing file **or** missing heading anchor) and + `LinkAmbiguous`, computed on the fly (model-free, no build required). +- `search`: `--connected-to [--depth N]`, `--backlinks-of ` (file- + or section-level), composed with the existing modes and `--where`. +- `info`: edge count, broken / ambiguous counts, top-N most-linked + files/sections. +- `[graph]` config (`enabled` default false, `resolve`, `strict`) and the + `is_reference` schema attribute. +- Multi-hop via petgraph projection, no cache. **Sections only** (blocks + deferred). + +Semantic `--related-to` and the discovery workflows are **not here** — they are +TODO-0171. Deferred, gated on evidence: a disposable traversal cache (only past +~100K files); graph-as-rank-boost. **`--connected-to --depth > 1` needs a +benchmark** on a 50K+ synthetic corpus before promotion. ## Interaction with other TODOs -- **TODO-0016 (Lance swap, done).** The `links` column lives on the existing chunk table — no new dataset, same single storage stack. -- **TODO-0173 (incremental writes, done).** The `links` column rides the existing incremental write path. -- **TODO-0171 (semantic-assisted link authoring).** Companion, reversed dependency: 0171's discovery workflows (`--related-to`, missing-links, clustering) *consume* this graph (to know what's already linked) and *feed* it (committed suggestions become edges here). Semantic never becomes an edge in this TODO. -- **TODO-0170 (incremental cache).** Only relevant if the deferred disposable traversal cache is ever built. -- **TODO-0156 (Array of structured items).** Sidestepped — links are an internal nested `List` storage column, not a user-facing `Array(Object)` frontmatter field. +- **TODO-0016 (Lance swap, done).** The `links` column lives on the existing + chunk table — no new dataset, same single storage stack. +- **TODO-0173 (incremental writes, done).** The `links` column rides the + existing incremental write path. +- **TODO-0171 (semantic-assisted link authoring).** Companion, reversed + dependency: 0171's discovery workflows (`--related-to`, missing-links, + clustering) _consume_ this graph (to know what's already linked) and _feed_ it + (committed suggestions become edges here). Semantic never becomes an edge in + this TODO. +- **TODO-0170 (incremental cache).** Only relevant if the deferred disposable + traversal cache is ever built. +- **TODO-0156 (Array of structured items).** Sidestepped — links are an internal + nested `List` storage column, not a user-facing `Array(Object)` + frontmatter field. ## Design history - **2026-03-14**: TODO opened with markdown links + wikilinks extraction goal. -- **2026-05-30**: Redesigned for Lance with denormalised List columns on chunk rows. TODO-0171 opened as companion. -- **2026-06-16 / 06-17**: Redesigned for a canonical edge table + custom CSR sidecar; adversarial review flagged the CSR scope as overreach and evaluated `lance-graph` (Cypher → DataFusion over Lance) as non-viable (bus factor of 1, locked to Lance 1.0.0, ~10× slower than Kuzu). `KuzuDB` / `CozoDB` are dead — the embedded columnar-graph category is in decline. -- **2026-06-19**: Layered shape — a canonical `edges.lance` table with petgraph at query time, redb/CSR adjacency as deferred escalations. -- **2026-06-25**: Superseded storage and UX. Dropped `edges.lance` for a `links` column on the chunk row; dissolved the standalone `mdvs graph` command (folds into `search` / `check` / `info`); moved semantic on-the-fly. Captured prior art (filter-vs-rank) and situated against graphify. -- **2026-07-03**: Cleaned up (removed the superseded `edges.lance` / phased-adjacency sections) and **finalized the graph model**: nodes = files **and** sections (internal heading hierarchy → Structural edges + broken-heading detection); both Obsidian and GitHub heading notations resolve to the same section node (one canonical slugger); the file/section × file/section link-type model; **semantic excluded from the graph** (discovery-only, moved to TODO-0171, which was reframed from a stored similarity-edge graph into semantic-assisted link-authoring workflows). Sections only; blocks deferred. +- **2026-05-30**: Redesigned for Lance with denormalised List columns on chunk + rows. TODO-0171 opened as companion. +- **2026-06-16 / 06-17**: Redesigned for a canonical edge table + custom CSR + sidecar; adversarial review flagged the CSR scope as overreach and evaluated + `lance-graph` (Cypher → DataFusion over Lance) as non-viable (bus factor of 1, + locked to Lance 1.0.0, ~10× slower than Kuzu). `KuzuDB` / `CozoDB` are dead — + the embedded columnar-graph category is in decline. +- **2026-06-19**: Layered shape — a canonical `edges.lance` table with petgraph + at query time, redb/CSR adjacency as deferred escalations. +- **2026-06-25**: Superseded storage and UX. Dropped `edges.lance` for a `links` + column on the chunk row; dissolved the standalone `mdvs graph` command (folds + into `search` / `check` / `info`); moved semantic on-the-fly. Captured prior + art (filter-vs-rank) and situated against graphify. +- **2026-07-03**: Cleaned up (removed the superseded `edges.lance` / + phased-adjacency sections) and **finalized the graph model**: nodes = files + **and** sections (internal heading hierarchy → Structural edges + + broken-heading detection); both Obsidian and GitHub heading notations resolve + to the same section node (one canonical slugger); the file/section × + file/section link-type model; **semantic excluded from the graph** + (discovery-only, moved to TODO-0171, which was reframed from a stored + similarity-edge graph into semantic-assisted link-authoring workflows). + Sections only; blocks deferred. diff --git a/docs/spec/todos/TODO-0107.md b/docs/spec/todos/TODO-0107.md index 4a49620..b32e226 100644 --- a/docs/spec/todos/TODO-0107.md +++ b/docs/spec/todos/TODO-0107.md @@ -21,24 +21,56 @@ files_updated: ## Resolution -Shipped **config + docs only** on `feat/pre-commit-hook` — no new CLI surface. Two consumer paths, both running `mdvs check --no-update`: - -- **pre-commit framework** — `.pre-commit-hooks.yaml` at the repo root declares `id: mdvs-check` (`language: system`, `types: [markdown]`, `pass_filenames: false`). Users reference it from their `.pre-commit-config.yaml`. -- **Plain git hook** — the `.git/hooks/pre-commit` → `mdvs check --no-update || exit 1` one-liner, documented for the zero-dependency case plus the `core.hooksPath` variant for sharing across a team. - -Documented in a new `book/src/recipes/pre-commit.md` (linked from SUMMARY.md and cross-linked from the existing `ci.md` pre-commit bullet). TODO-0190 had already shipped a full pre-commit walkthrough inside `book/src/recipes/agent-harnesses.md`; that section is now cut down to what a pre-commit hook *is* and why it's the harness-independent net, deferring setup to the new page. Its `#pre-commit-hook` anchor is preserved — four per-harness pages deep-link to it. The PATH-gotcha and `language: rust` notes moved to the new page rather than being dropped, and the config it recommends switched from a hand-written `repo: local` block to referencing the shipped `.pre-commit-hooks.yaml`, which did not exist when 0190 was written. - -Improves on the original draft below in one place: the hook runs `mdvs check --no-update`, not bare `mdvs check` — the same determinism argument the CI recipe makes (a new field must be added to the schema deliberately, not silently absorbed by re-inference). +Shipped **config + docs only** on `feat/pre-commit-hook` — no new CLI surface. +Two consumer paths, both running `mdvs check --no-update`: + +- **pre-commit framework** — `.pre-commit-hooks.yaml` at the repo root declares + `id: mdvs-check` (`language: system`, `types: [markdown]`, + `pass_filenames: false`). Users reference it from their + `.pre-commit-config.yaml`. +- **Plain git hook** — the `.git/hooks/pre-commit` → + `mdvs check --no-update || exit 1` one-liner, documented for the + zero-dependency case plus the `core.hooksPath` variant for sharing across a + team. + +Documented in a new `book/src/recipes/pre-commit.md` (linked from SUMMARY.md and +cross-linked from the existing `ci.md` pre-commit bullet). TODO-0190 had already +shipped a full pre-commit walkthrough inside +`book/src/recipes/agent-harnesses.md`; that section is now cut down to what a +pre-commit hook _is_ and why it's the harness-independent net, deferring setup +to the new page. Its `#pre-commit-hook` anchor is preserved — four per-harness +pages deep-link to it. The PATH-gotcha and `language: rust` notes moved to the +new page rather than being dropped, and the config it recommends switched from a +hand-written `repo: local` block to referencing the shipped +`.pre-commit-hooks.yaml`, which did not exist when 0190 was written. + +Improves on the original draft below in one place: the hook runs +`mdvs check --no-update`, not bare `mdvs check` — the same determinism argument +the CI recipe makes (a new field must be added to the schema deliberately, not +silently absorbed by re-inference). ### The `mdvs hook install` subcommand — dropped, superseded by TODO-0190 -The "Optional `mdvs hook` subcommand" sketched below (`mdvs hook install` / `uninstall` to write `.git/hooks/pre-commit`) is **not** shipped, and its proposed name is now unavailable. [TODO-0190](TODO-0190.md) shipped `mdvs scaffold` + `mdvs hook handle` and **claimed the `mdvs hook` namespace for agent-harness PostToolUse hooks** — a different meaning of "hook" (agent tool-call, not git commit). A git-hook installer under `mdvs hook install` would overload that namespace confusingly. - -If a convenience installer is ever wanted, the right home is the existing 0190 taxonomy — `mdvs scaffold git-hook`, a sibling of `mdvs scaffold {skill,snippet,hook}` — not a fresh `mdvs hook` verb. That was scoped out of this branch by explicit decision (2026-07-10): config + docs cover the need, and 0107's own "start simple" plan already deferred the subcommand. Left unbuilt on purpose; revisit only if users ask for one-command install. +The "Optional `mdvs hook` subcommand" sketched below (`mdvs hook install` / +`uninstall` to write `.git/hooks/pre-commit`) is **not** shipped, and its +proposed name is now unavailable. [TODO-0190](TODO-0190.md) shipped +`mdvs scaffold` + `mdvs hook handle` and **claimed the `mdvs hook` namespace for +agent-harness PostToolUse hooks** — a different meaning of "hook" (agent +tool-call, not git commit). A git-hook installer under `mdvs hook install` would +overload that namespace confusingly. + +If a convenience installer is ever wanted, the right home is the existing 0190 +taxonomy — `mdvs scaffold git-hook`, a sibling of +`mdvs scaffold {skill,snippet,hook}` — not a fresh `mdvs hook` verb. That was +scoped out of this branch by explicit decision (2026-07-10): config + docs cover +the need, and 0107's own "start simple" plan already deferred the subcommand. +Left unbuilt on purpose; revisit only if users ask for one-command install. ## Summary -Provide a pre-commit hook that runs `mdvs check` before each commit, catching frontmatter violations before they reach the repo. Support both native git hooks and the [pre-commit](https://pre-commit.com/) framework. +Provide a pre-commit hook that runs `mdvs check` before each commit, catching +frontmatter violations before they reach the repo. Support both native git hooks +and the [pre-commit](https://pre-commit.com/) framework. ## Details @@ -51,11 +83,15 @@ A simple shell script users can copy to `.git/hooks/pre-commit`: mdvs check || exit 1 ``` -This is zero-dependency — just needs `mdvs` on `PATH`. Could ship as `hooks/pre-commit` in the repo or be generated by an `mdvs hook install` command. +This is zero-dependency — just needs `mdvs` on `PATH`. Could ship as +`hooks/pre-commit` in the repo or be generated by an `mdvs hook install` +command. ### pre-commit framework -The [pre-commit](https://pre-commit.com/) framework is widely used and supports declaring hooks in a `.pre-commit-hooks.yaml` at the repo root. Users add mdvs to their `.pre-commit-config.yaml`: +The [pre-commit](https://pre-commit.com/) framework is widely used and supports +declaring hooks in a `.pre-commit-hooks.yaml` at the repo root. Users add mdvs +to their `.pre-commit-config.yaml`: ```yaml repos: @@ -76,7 +112,9 @@ This requires adding a `.pre-commit-hooks.yaml` to the mdvs repo: pass_filenames: false ``` -`language: system` means pre-commit expects `mdvs` to be installed already (no automatic install from source). `pass_filenames: false` because `mdvs check` operates on the whole directory, not individual files. +`language: system` means pre-commit expects `mdvs` to be installed already (no +automatic install from source). `pass_filenames: false` because `mdvs check` +operates on the whole directory, not individual files. ### `mdvs hook` subcommand (optional) @@ -87,18 +125,26 @@ mdvs hook install # writes .git/hooks/pre-commit mdvs hook uninstall # removes it ``` -This avoids users having to manually copy scripts. Should check for existing hooks and offer to append rather than overwrite. +This avoids users having to manually copy scripts. Should check for existing +hooks and offer to append rather than overwrite. ### Design questions -- Should the hook run on the whole directory or only on staged `.md` files? Whole-directory is simpler and catches cross-file violations (e.g., required fields). Staged-only is faster but misses context. -- Should we support `language: rust` in pre-commit (auto-compile from source)? Slow on first run but zero-install. `language: system` is faster but requires manual install. -- Should the hook respect `--where` or other filtering? Probably not — keep it simple, just `mdvs check`. -- Should there be a `--hook` flag on `mdvs check` that adjusts output for hook context (e.g., shorter messages, exit codes)? +- Should the hook run on the whole directory or only on staged `.md` files? + Whole-directory is simpler and catches cross-file violations (e.g., required + fields). Staged-only is faster but misses context. +- Should we support `language: rust` in pre-commit (auto-compile from source)? + Slow on first run but zero-install. `language: system` is faster but requires + manual install. +- Should the hook respect `--where` or other filtering? Probably not — keep it + simple, just `mdvs check`. +- Should there be a `--hook` flag on `mdvs check` that adjusts output for hook + context (e.g., shorter messages, exit codes)? ### Approach Start simple: + 1. Add `.pre-commit-hooks.yaml` to the repo (`language: system`) 2. Document the native git hook one-liner in the book 3. Defer `mdvs hook install` subcommand to later @@ -106,4 +152,5 @@ Start simple: ## Files - `.pre-commit-hooks.yaml` — pre-commit framework hook definition -- `book/src/recipes/` — document hook setup (could be its own page or part of CI recipe) +- `book/src/recipes/` — document hook setup (could be its own page or part of CI + recipe) diff --git a/docs/spec/todos/TODO-0108.md b/docs/spec/todos/TODO-0108.md index 016d983..103fb9c 100644 --- a/docs/spec/todos/TODO-0108.md +++ b/docs/spec/todos/TODO-0108.md @@ -14,11 +14,16 @@ files_updated: [src/cmd/build.rs] ## Summary -`mdvs build --set-revision "" --force` writes `revision = ""` to `mdvs.toml` instead of removing the field. Both `""` and `"None"` should be treated as "unset" and remove the `revision` key entirely (relying on `#[serde(skip_serializing_if = "Option::is_none")]`). +`mdvs build --set-revision "" --force` writes `revision = ""` to `mdvs.toml` +instead of removing the field. Both `""` and `"None"` should be treated as +"unset" and remove the `revision` key entirely (relying on +`#[serde(skip_serializing_if = "Option::is_none")]`). ## Details -Currently `--set-revision` takes a `String` and stores it as `Some(value)`. When the value is empty or literally `"None"`, it should map to `None` so serde skips serialization. +Currently `--set-revision` takes a `String` and stores it as `Some(value)`. When +the value is empty or literally `"None"`, it should map to `None` so serde skips +serialization. ## Files diff --git a/docs/spec/todos/TODO-0109.md b/docs/spec/todos/TODO-0109.md index e85f222..df5fadd 100644 --- a/docs/spec/todos/TODO-0109.md +++ b/docs/spec/todos/TODO-0109.md @@ -12,15 +12,21 @@ blocks: [] ## Summary -When `--where` references an invalid column name, DataFusion's error message leaks internal table aliases (e.g., `No field named _filename. Did you mean 'f.filepath'?`). The `f.` prefix is the internal SQL alias for `files_v` — users shouldn't see it. +When `--where` references an invalid column name, DataFusion's error message +leaks internal table aliases (e.g., +`No field named _filename. Did you mean 'f.filepath'?`). The `f.` prefix is the +internal SQL alias for `files_v` — users shouldn't see it. ## Details -Intercept DataFusion schema errors during search and rewrite them with user-friendly messages. For example: +Intercept DataFusion schema errors during search and rewrite them with +user-friendly messages. For example: -- `No field named _filename. Did you mean 'f.filepath'?` → `No field named '_filename'. Did you mean 'filepath'?` +- `No field named _filename. Did you mean 'f.filepath'?` → + `No field named '_filename'. Did you mean 'filepath'?` - Strip `f.` and `c.` table alias prefixes from suggestions ## Files -- `src/pipeline/execute_search.rs` or `src/cmd/search.rs` — intercept and rewrite DataFusion errors before displaying +- `src/pipeline/execute_search.rs` or `src/cmd/search.rs` — intercept and + rewrite DataFusion errors before displaying diff --git a/docs/spec/todos/TODO-0110.md b/docs/spec/todos/TODO-0110.md index 5b98938..b3877b2 100644 --- a/docs/spec/todos/TODO-0110.md +++ b/docs/spec/todos/TODO-0110.md @@ -14,17 +14,32 @@ superseded_by: 119 ## Summary -Rework the pipeline output model so that a processing step can itself contain child steps, forming a tree. This supports commands that auto-run upstream commands (TODO-0099): search embeds build, build embeds update, check embeds update. The output tree mirrors the execution tree, enabling a unified rendering rule: compact mode shows "surprising" (auto-triggered) steps as summary lines, verbose mode expands the full tree. +Rework the pipeline output model so that a processing step can itself contain +child steps, forming a tree. This supports commands that auto-run upstream +commands (TODO-0099): search embeds build, build embeds update, check embeds +update. The output tree mirrors the execution tree, enabling a unified rendering +rule: compact mode shows "surprising" (auto-triggered) steps as summary lines, +verbose mode expands the full tree. ## Problem -Today each command has a flat `ProcessOutput` struct with N fields of type `ProcessingStepResult`. Init and update duplicate build's entire pipeline inline and flatten everything into one struct. After TODO-0099, the duplication gets worse: search would need to carry build's steps and update's steps alongside its own core steps, all flattened. Check, build, and search all gain auto-update capability, compounding the problem. +Today each command has a flat `ProcessOutput` struct with N fields of type +`ProcessingStepResult`. Init and update duplicate build's entire pipeline +inline and flatten everything into one struct. After TODO-0099, the duplication +gets worse: search would need to carry build's steps and update's steps +alongside its own core steps, all flattened. Check, build, and search all gain +auto-update capability, compounding the problem. -The flat model also can't express which steps were auto-triggered vs core. There's no structural distinction between "search loaded a model" (expected) and "search auto-built an index" (surprising). This distinction is needed for the compact rendering rule. +The flat model also can't express which steps were auto-triggered vs core. +There's no structural distinction between "search loaded a model" (expected) and +"search auto-built an index" (surprising). This distinction is needed for the +compact rendering rule. ## Proposed model -A step is either a **leaf** (atomic operation like scan, embed, write_index) or a **composite** (a named group of child steps, representing an auto-triggered upstream command). +A step is either a **leaf** (atomic operation like scan, embed, write_index) or +a **composite** (a named group of child steps, representing an auto-triggered +upstream command). ### Output tree by command @@ -91,19 +106,31 @@ clean ### Composite steps reuse command output structs -`BuildCommandOutput` already contains `process: BuildProcessOutput` and `result: Option`. Search's `auto_build` field would be `Option` — the same struct build uses as its top-level output. Same for build's `auto_update` being `Option`. +`BuildCommandOutput` already contains `process: BuildProcessOutput` and +`result: Option`. Search's `auto_build` field would be +`Option` — the same struct build uses as its top-level +output. Same for build's `auto_update` being `Option`. This means: + - No new intermediate types needed -- Build's output struct works identically whether it's the top-level command or embedded inside search -- The tree is naturally recursive through composition (not via a generic recursive type) +- Build's output struct works identically whether it's the top-level command or + embedded inside search +- The tree is naturally recursive through composition (not via a generic + recursive type) ### Step deduplication -When search auto-builds (and build auto-updates), some steps overlap conceptually (read_config, scan, load_model). The upstream execution produces results that the downstream command reuses. Implementation options: +When search auto-builds (and build auto-updates), some steps overlap +conceptually (read_config, scan, load_model). The upstream execution produces +results that the downstream command reuses. Implementation options: -1. **Upstream runs, downstream skips** — if auto_build ran and loaded the model, search's own load_model is Skipped (model already in memory). The output tree shows load_model under auto_build as Completed and under search as Skipped. -2. **Downstream omits the step entirely** — search's ProcessOutput doesn't even have a load_model field when auto_build is present. This changes the struct shape based on runtime state, which is awkward in Rust. +1. **Upstream runs, downstream skips** — if auto_build ran and loaded the model, + search's own load_model is Skipped (model already in memory). The output tree + shows load_model under auto_build as Completed and under search as Skipped. +2. **Downstream omits the step entirely** — search's ProcessOutput doesn't even + have a load_model field when auto_build is present. This changes the struct + shape based on runtime state, which is awkward in Rust. Option 1 is simpler and already supported by `ProcessingStepResult::Skipped`. @@ -113,19 +140,23 @@ The rendering tree is governed by two rules: ### Rule 1: Compact shows surprising, verbose shows all -| Mode | Core steps | Auto steps | -|---|---|---| -| Compact | Hidden | Shown as summary line | +| Mode | Core steps | Auto steps | +| ------- | ------------------- | -------------------------- | +| Compact | Hidden | Shown as summary line | | Verbose | Shown as step lines | Shown as expanded sub-tree | -| Error | Shown up to failure | Shown up to failure | +| Error | Shown up to failure | Shown up to failure | ### Rule 2: Compact/verbose propagates into composites -When rendering an auto step in verbose mode, the composite's own verbose rendering is used. When rendering in compact mode, the composite produces a single summary line from its result (e.g., "Auto-built index (46 files, 312 chunks)"). +When rendering an auto step in verbose mode, the composite's own verbose +rendering is used. When rendering in compact mode, the composite produces a +single summary line from its result (e.g., "Auto-built index (46 files, 312 +chunks)"). ### Text examples **Compact search (auto_build fired, which auto_updated):** + ``` Auto-updated schema (0 changes) Built index (46 files, 312 chunks) @@ -136,6 +167,7 @@ Searched "rust" — 5 hits ``` **Verbose search (same scenario):** + ``` Auto-update: Read config: mdvs.toml @@ -162,6 +194,7 @@ Searched "rust" — 5 hits ``` **Compact search (no auto steps):** + ``` Searched "rust" — 5 hits @@ -171,12 +204,14 @@ Searched "rust" — 5 hits No preamble lines — looks exactly like today. **Compact check (auto_update fired):** + ``` Auto-updated schema (1 added, 0 changed) Checked 46 files — 0 violations ``` **Compact check (no auto steps):** + ``` Checked 46 files — 0 violations ``` @@ -184,6 +219,7 @@ Checked 46 files — 0 violations ### JSON examples **Compact JSON (auto_build fired):** + ```json { "result": { @@ -196,6 +232,7 @@ Checked 46 files — 0 violations Compact JSON remains result-only (existing behavior). Auto steps are not shown. **Verbose JSON (auto_build fired):** + ```json { "process": { @@ -224,35 +261,52 @@ Compact JSON remains result-only (existing behavior). Auto steps are not shown. } ``` -The tree structure is self-documenting: `auto_build.process.auto_update` is present if and only if update ran inside build inside search. +The tree structure is self-documenting: `auto_build.process.auto_update` is +present if and only if update ran inside build inside search. ## Resolution -Superseded by [TODO-0119](TODO-0119.md). TODO-0110 proposed composing existing `*CommandOutput` structs for nesting. TODO-0119 replaces the entire output model with a unified `Step` tree where both leaf steps and commands are the same node type, with an `Outcome` enum and recursive rendering. +Superseded by [TODO-0119](TODO-0119.md). TODO-0110 proposed composing existing +`*CommandOutput` structs for nesting. TODO-0119 replaces the entire output model +with a unified `Step` tree where both leaf steps and commands are the same node +type, with an `Outcome` enum and recursive rendering. ## Original Implementation scope ### Pipeline framework (`src/pipeline/mod.rs`) -No changes to `ProcessingStepResult`, `ProcessingStep`, or `StepOutput`. These remain leaf-level types. The nesting is achieved by composing existing `*CommandOutput` structs, not by making the framework itself recursive. +No changes to `ProcessingStepResult`, `ProcessingStep`, or `StepOutput`. These +remain leaf-level types. The nesting is achieved by composing existing +`*CommandOutput` structs, not by making the framework itself recursive. ### Command output structs -- **`CheckCommandOutput`** — add `auto_update: Option` field to `CheckProcessOutput` -- **`BuildCommandOutput`** — add `auto_update: Option` field to `BuildProcessOutput` -- **`SearchCommandOutput`** — add `auto_build: Option` field to `SearchProcessOutput` -- **`UpdateCommandOutput`**, **`InitCommandOutput`**, etc. — unchanged (they don't auto-run anything) +- **`CheckCommandOutput`** — add `auto_update: Option` + field to `CheckProcessOutput` +- **`BuildCommandOutput`** — add `auto_update: Option` + field to `BuildProcessOutput` +- **`SearchCommandOutput`** — add `auto_build: Option` field + to `SearchProcessOutput` +- **`UpdateCommandOutput`**, **`InitCommandOutput`**, etc. — unchanged (they + don't auto-run anything) ### CommandOutput trait -- `format_text(&self, verbose: bool)` — existing signature works. Each command's impl checks for auto steps and renders them according to the rules. -- `format_json(&self, verbose: bool)` — existing `format_json_compact` helper works. Auto steps are included in the full struct (verbose) and excluded from result-only (compact). -- Add a helper method like `format_summary(&self) -> String` on result types that produces the one-line summary for compact rendering of auto steps. +- `format_text(&self, verbose: bool)` — existing signature works. Each command's + impl checks for auto steps and renders them according to the rules. +- `format_json(&self, verbose: bool)` — existing `format_json_compact` helper + works. Auto steps are included in the full struct (verbose) and excluded from + result-only (compact). +- Add a helper method like `format_summary(&self) -> String` on result types + that produces the one-line summary for compact rendering of auto steps. ### Rendering helpers Add to `src/output.rs` or `src/table.rs`: -- `format_auto_step(label: &str, output: &impl CommandOutput, verbose: bool) -> String` — renders a composite step as either a summary line (compact) or an indented sub-tree (verbose) + +- `format_auto_step(label: &str, output: &impl CommandOutput, verbose: bool) -> String` + — renders a composite step as either a summary line (compact) or an indented + sub-tree (verbose) ### Files affected @@ -262,23 +316,32 @@ Add to `src/output.rs` or `src/table.rs`: - `src/cmd/build.rs` — add `auto_update` to process output, rendering logic - `src/cmd/search.rs` — add `auto_build` to process output, rendering logic - `src/cmd/update.rs` — add `format_summary()` to result type -- `src/cmd/init.rs` — remove inline build pipeline (covered by TODO-0099, but output struct simplifies) +- `src/cmd/init.rs` — remove inline build pipeline (covered by TODO-0099, but + output struct simplifies) ### Current state (from TODO-0099) -TODO-0099 implemented auto-update/auto-build by calling `update::run()` and `build::run()` from downstream commands and nesting the full `*CommandOutput` structs: +TODO-0099 implemented auto-update/auto-build by calling `update::run()` and +`build::run()` from downstream commands and nesting the full `*CommandOutput` +structs: - `CheckProcessOutput.auto_update: Option` - `BuildProcessOutput.auto_update: Option` - `SearchProcessOutput.auto_build: Option` (planned) This works functionally but is a naive first pass. TODO-0110 should: -1. Evaluate whether to keep the nesting or extract a helper function (e.g., `update::run_core()`) that returns a simpler summary -2. Define proper rendering rules for the nested output (compact summary vs verbose tree) + +1. Evaluate whether to keep the nesting or extract a helper function (e.g., + `update::run_core()`) that returns a simpler summary +2. Define proper rendering rules for the nested output (compact summary vs + verbose tree) 3. Ensure JSON output structure is clean and self-documenting ### Interaction with other TODOs -- **TODO-0099** (prerequisite): defines which commands auto-run which. Done — nesting already exists. -- **TODO-0100** (blocked): text output redesign builds on this nested structure. The rendering rules defined here are the foundation. -- **TODO-0101** (blocked by 0100): markdown format follows the same rendering rules, just different table style. +- **TODO-0099** (prerequisite): defines which commands auto-run which. Done — + nesting already exists. +- **TODO-0100** (blocked): text output redesign builds on this nested structure. + The rendering rules defined here are the foundation. +- **TODO-0101** (blocked by 0100): markdown format follows the same rendering + rules, just different table style. diff --git a/docs/spec/todos/TODO-0111.md b/docs/spec/todos/TODO-0111.md index 8f05fda..99e2374 100644 --- a/docs/spec/todos/TODO-0111.md +++ b/docs/spec/todos/TODO-0111.md @@ -14,7 +14,10 @@ files_updated: [src/schema/config.rs, src/schema/shared.rs] ## Summary -Add `#[serde(deny_unknown_fields)]` to all config structs in `src/schema/config.rs` and `src/schema/shared.rs`. Today, unknown keys in `mdvs.toml` are silently ignored during deserialization. This masks typos (e.g., `autobuild` instead of `auto_build`) and stale keys from config migrations. +Add `#[serde(deny_unknown_fields)]` to all config structs in +`src/schema/config.rs` and `src/schema/shared.rs`. Today, unknown keys in +`mdvs.toml` are silently ignored during deserialization. This masks typos (e.g., +`autobuild` instead of `auto_build`) and stale keys from config migrations. ## Details @@ -34,20 +37,32 @@ Every struct that maps to a TOML section needs `#[serde(deny_unknown_fields)]`: ### Error message -Serde's default error for unknown fields is clear: `"unknown field 'foo', expected one of ..."`. This surfaces through `anyhow::Result` in `MdvsToml::read()` and becomes a `ProcessingStepError` with `ErrorKind::User` in `run_read_config()`. No extra wrapping needed. +Serde's default error for unknown fields is clear: +`"unknown field 'foo', expected one of ..."`. This surfaces through +`anyhow::Result` in `MdvsToml::read()` and becomes a `ProcessingStepError` with +`ErrorKind::User` in `run_read_config()`. No extra wrapping needed. ### Migration impact -This is a breaking change for users with stale keys in their toml. Most importantly, TODO-0099 removes `auto_build` from `[update]` and `--suppress-auto-build`/`--model`/`--revision`/`--chunk-size` from init. Users with existing `[update].auto_build` will get a hard error after both changes land. +This is a breaking change for users with stale keys in their toml. Most +importantly, TODO-0099 removes `auto_build` from `[update]` and +`--suppress-auto-build`/`--model`/`--revision`/`--chunk-size` from init. Users +with existing `[update].auto_build` will get a hard error after both changes +land. -**This TODO should land before TODO-0099** so that users get clean errors immediately, and TODO-0099's config changes are automatically enforced. +**This TODO should land before TODO-0099** so that users get clean errors +immediately, and TODO-0099's config changes are automatically enforced. ### Testing - Add tests for each section with an unknown key → verify deserialization error -- Add a test with a typo'd key (e.g., `glob_pattern` instead of `glob`) → verify error message includes the expected field names +- Add a test with a typo'd key (e.g., `glob_pattern` instead of `glob`) → verify + error message includes the expected field names ### Files -- `src/schema/config.rs` — add `deny_unknown_fields` to `MdvsToml`, `ScanConfig`, `UpdateConfig`, `FieldsConfig`, `TomlField`, `SearchConfig`, `StorageConfig` -- `src/schema/shared.rs` — add `deny_unknown_fields` to `EmbeddingModelConfig`, `ChunkingConfig` +- `src/schema/config.rs` — add `deny_unknown_fields` to `MdvsToml`, + `ScanConfig`, `UpdateConfig`, `FieldsConfig`, `TomlField`, `SearchConfig`, + `StorageConfig` +- `src/schema/shared.rs` — add `deny_unknown_fields` to `EmbeddingModelConfig`, + `ChunkingConfig` diff --git a/docs/spec/todos/TODO-0112.md b/docs/spec/todos/TODO-0112.md index f0f5483..4c796bc 100644 --- a/docs/spec/todos/TODO-0112.md +++ b/docs/spec/todos/TODO-0112.md @@ -12,26 +12,37 @@ blocks: [] ## Summary -The book doesn't explain what `--output json` produces. Users need to know the JSON structure to use mdvs in scripts, CI, or pipelines. Find a way to integrate this meaningfully — not just a dump of the schema, but practical guidance. +The book doesn't explain what `--output json` produces. Users need to know the +JSON structure to use mdvs in scripts, CI, or pipelines. Find a way to integrate +this meaningfully — not just a dump of the schema, but practical guidance. ## Details -Currently the book mentions `--output json` only in the global flags table on the configuration page. No examples, no schema, no explanation of compact vs verbose differences. +Currently the book mentions `--output json` only in the global flags table on +the configuration page. No examples, no schema, no explanation of compact vs +verbose differences. ### What to document - Compact JSON (`-o json`): result only, e.g. `{"result": {...}}` -- Verbose JSON (`-o json -v`): includes `process` steps with elapsed times + full result +- Verbose JSON (`-o json -v`): includes `process` steps with elapsed times + + full result - JSON structure per command (what fields are present) - Practical examples: piping to `jq`, using in scripts ### Placement options -- **Per-command**: add a "JSON output" subsection to each command page with an example. Keeps everything together but adds bulk to every page. -- **Dedicated page**: a "JSON Output" reference page under Reference, covering the structure once and linking from command pages. DRY but one more page to navigate. -- **Configuration page**: expand the global flags section with JSON examples. Lightweight but may get long. -- **Hybrid**: brief `--output json` example on each command page (one-liner), full reference on a dedicated page. +- **Per-command**: add a "JSON output" subsection to each command page with an + example. Keeps everything together but adds bulk to every page. +- **Dedicated page**: a "JSON Output" reference page under Reference, covering + the structure once and linking from command pages. DRY but one more page to + navigate. +- **Configuration page**: expand the global flags section with JSON examples. + Lightweight but may get long. +- **Hybrid**: brief `--output json` example on each command page (one-liner), + full reference on a dedicated page. ### Depends on -TODO-0100 (output redesign) will change the JSON structure, so this should be written after that's done — or written now with a note that it will change. +TODO-0100 (output redesign) will change the JSON structure, so this should be +written after that's done — or written now with a note that it will change. diff --git a/docs/spec/todos/TODO-0113.md b/docs/spec/todos/TODO-0113.md index da37087..b508e49 100644 --- a/docs/spec/todos/TODO-0113.md +++ b/docs/spec/todos/TODO-0113.md @@ -12,18 +12,28 @@ blocks: [] ## Summary -The first `build` or `search` run downloads the embedding model from HuggingFace, which can take several seconds to minutes depending on model size and network speed. There's no visual feedback during the download — the CLI appears frozen. Add a progress bar (similar to Python's tqdm or Rust's indicatif) for model download and optionally for embedding. +The first `build` or `search` run downloads the embedding model from +HuggingFace, which can take several seconds to minutes depending on model size +and network speed. There's no visual feedback during the download — the CLI +appears frozen. Add a progress bar (similar to Python's tqdm or Rust's +indicatif) for model download and optionally for embedding. ## Details ### Where progress is needed -1. **Model download** — the HuggingFace model download via `model2vec-rs`. This is the most important one — it's the longest wait and happens on first run with no feedback. -2. **Embedding** — chunking and embedding files. For large vaults this can take a few seconds. Less critical since it's fast, but a progress indicator would be nice. +1. **Model download** — the HuggingFace model download via `model2vec-rs`. This + is the most important one — it's the longest wait and happens on first run + with no feedback. +2. **Embedding** — chunking and embedding files. For large vaults this can take + a few seconds. Less critical since it's fast, but a progress indicator would + be nice. ### Approach -Use the `indicatif` crate (standard Rust progress bar library). Key features needed: +Use the `indicatif` crate (standard Rust progress bar library). Key features +needed: + - Download progress bar with bytes/total, speed, ETA - Spinner or progress bar for embedding (files processed / total) - Output to stderr (stdout is reserved for command output) @@ -33,7 +43,8 @@ Use the `indicatif` crate (standard Rust progress bar library). Key features nee - `model2vec-rs` may not expose download progress callbacks — check the API - If not, we may need to wrap the download with our own HTTP client -- The progress bar must not interfere with the structured output model (ProcessingStepResult) +- The progress bar must not interfere with the structured output model + (ProcessingStepResult) ## Files diff --git a/docs/spec/todos/TODO-0114.md b/docs/spec/todos/TODO-0114.md index 0e82c30..8d1e592 100644 --- a/docs/spec/todos/TODO-0114.md +++ b/docs/spec/todos/TODO-0114.md @@ -12,59 +12,75 @@ blocks: [] ## Summary -Keep book output examples in sync with actual CLI behavior. A generation script runs `mdvs` commands against `example_kb/`, saves stdout to `book/src/generated/`, and book pages pull them in with `{{#include generated/...}}`. +Keep book output examples in sync with actual CLI behavior. A generation script +runs `mdvs` commands against `example_kb/`, saves stdout to +`book/src/generated/`, and book pages pull them in with +`{{#include generated/...}}`. ## Approach -**Include-file pattern** (not mdbook-cmdrun): a shell script generates output files, book pages include them. Simpler, no extra preprocessor dependency, works with plain `mdbook build`. +**Include-file pattern** (not mdbook-cmdrun): a shell script generates output +files, book pages include them. Simpler, no extra preprocessor dependency, works +with plain `mdbook build`. ### Generated files -- Live in `book/src/generated/` (committed, so local `mdbook build` works without running the script) -- Naming: `-.txt` (e.g. `init-compact.txt`, `check-violations.txt`, `search-where.txt`) +- Live in `book/src/generated/` (committed, so local `mdbook build` works + without running the script) +- Naming: `-.txt` (e.g. `init-compact.txt`, + `check-violations.txt`, `search-where.txt`) - Script is idempotent — re-running produces identical output if nothing changed - PRs that change output show diffs in the generated files ### Error examples Some pages show violation/error output. Two options: -1. **Fixture directory** (`book/fixtures/`) with markdown files + mdvs.toml designed to trigger specific violations -2. **Temp manipulation** in the script (copy example_kb, break something, capture output, clean up) -Option 1 is more maintainable — the fixtures are version-controlled and reviewable. +1. **Fixture directory** (`book/fixtures/`) with markdown files + mdvs.toml + designed to trigger specific violations +2. **Temp manipulation** in the script (copy example_kb, break something, + capture output, clean up) + +Option 1 is more maintainable — the fixtures are version-controlled and +reviewable. ### Embedding model in CI -Search/build/info-with-index examples need the model (~30MB). CI caches it between runs. Validation-layer commands (init, check, update, clean) don't need it. +Search/build/info-with-index examples need the model (~30MB). CI caches it +between runs. Validation-layer commands (init, check, update, clean) don't need +it. ## Pages with output examples -| Page | Commands shown | -|------|---------------| -| `commands/init.md` | init compact, verbose, dry-run | -| `commands/check.md` | check clean, with violations, verbose | -| `commands/build.md` | build compact, verbose, incremental | -| `commands/search.md` | search basic, --where, verbose | -| `commands/update.md` | update compact, verbose, dry-run | -| `commands/info.md` | info compact, verbose | -| `commands/clean.md` | clean compact, verbose | -| `introduction.md` | init | -| `getting-started.md` | init, check, search | -| `search-guide.md` | search with various --where filters | -| `concepts/validation.md` | check with violations | -| `recipes/obsidian.md` | init, check | +| Page | Commands shown | +| ------------------------ | ------------------------------------- | +| `commands/init.md` | init compact, verbose, dry-run | +| `commands/check.md` | check clean, with violations, verbose | +| `commands/build.md` | build compact, verbose, incremental | +| `commands/search.md` | search basic, --where, verbose | +| `commands/update.md` | update compact, verbose, dry-run | +| `commands/info.md` | info compact, verbose | +| `commands/clean.md` | clean compact, verbose | +| `introduction.md` | init | +| `getting-started.md` | init, check, search | +| `search-guide.md` | search with various --where filters | +| `concepts/validation.md` | check with violations | +| `recipes/obsidian.md` | init, check | ## Plan ### Wave 1: Infrastructure + - [ ] Create `scripts/generate-book-examples.sh` - [ ] Create `book/src/generated/` directory - [ ] Create `book/fixtures/` with markdown files + mdvs.toml for error examples -- [ ] Generate validation-layer outputs (init, check clean, check violations, update, clean, info without index) +- [ ] Generate validation-layer outputs (init, check clean, check violations, + update, clean, info without index) - [ ] Generate search-layer outputs (build, search, info with index) - [ ] Verify all generated files match current CLI output ### Wave 2: Update command pages + - [ ] `commands/init.md` — replace static blocks with `{{#include}}` - [ ] `commands/check.md` - [ ] `commands/build.md` @@ -75,6 +91,7 @@ Search/build/info-with-index examples need the model (~30MB). CI caches it betwe - [ ] Verify `mdbook build` produces correct HTML ### Wave 3: Update non-command pages + - [ ] `introduction.md` - [ ] `getting-started.md` - [ ] `search-guide.md` @@ -83,10 +100,12 @@ Search/build/info-with-index examples need the model (~30MB). CI caches it betwe - [ ] Verify `mdbook build` produces correct HTML ### Wave 4: CI integration + - [ ] Update `.github/workflows/book.yml` to build mdvs binary - [ ] Add model download + caching step - [ ] Run `scripts/generate-book-examples.sh` before `mdbook build` -- [ ] Optional: CI check that generated files are up-to-date (fail if script produces different output than committed) +- [ ] Optional: CI check that generated files are up-to-date (fail if script + produces different output than committed) ## Files diff --git a/docs/spec/todos/TODO-0115.md b/docs/spec/todos/TODO-0115.md index b0d4683..8576de8 100644 --- a/docs/spec/todos/TODO-0115.md +++ b/docs/spec/todos/TODO-0115.md @@ -12,7 +12,8 @@ blocks: [] ## Summary -Record terminal sessions with asciinema and embed them in mdBook pages. Gives readers a feel for the CLI workflow without requiring a live backend. +Record terminal sessions with asciinema and embed them in mdBook pages. Gives +readers a feel for the CLI workflow without requiring a live backend. ## Details @@ -25,7 +26,8 @@ Record terminal sessions with asciinema and embed them in mdBook pages. Gives re ### Embedding - Use the asciinema player (JavaScript widget) to embed recordings in book pages -- Add the player JS/CSS via `output.html.additional-js` and `output.html.additional-css` in `book.toml` +- Add the player JS/CSS via `output.html.additional-js` and + `output.html.additional-css` in `book.toml` - Embed with HTML in markdown: ```html @@ -43,15 +45,18 @@ Record terminal sessions with asciinema and embed them in mdBook pages. Gives re - **Getting started**: `mdvs init` → `mdvs search` (instant search workflow) - **Validation**: `mdvs init` → `mdvs check` (validation-only workflow) -- **Customized build**: `mdvs init` → `mdvs build --set-model ...` → `mdvs search` -- **Incremental build**: edit a file → `mdvs build` (shows only changed files re-embedded) +- **Customized build**: `mdvs init` → `mdvs build --set-model ...` → + `mdvs search` +- **Incremental build**: edit a file → `mdvs build` (shows only changed files + re-embedded) ### Considerations - Recordings need to be re-recorded when output format changes significantly - Player assets can be loaded from CDN or vendored locally - `.cast` files are small (text-based JSON) — fine to commit to repo -- Consider adding a `scripts/record-demos.sh` helper for reproducible recordings against `example_kb/` +- Consider adding a `scripts/record-demos.sh` helper for reproducible recordings + against `example_kb/` ### Files diff --git a/docs/spec/todos/TODO-0116.md b/docs/spec/todos/TODO-0116.md index 9922e15..1d31457 100644 --- a/docs/spec/todos/TODO-0116.md +++ b/docs/spec/todos/TODO-0116.md @@ -12,7 +12,8 @@ blocks: [] ## Summary -DataFusion is pulled with all default features, many of which mdvs doesn't use. Disabling unused features should reduce compile time and binary size. +DataFusion is pulled with all default features, many of which mdvs doesn't use. +Disabling unused features should reduce compile time and binary size. ## Details @@ -20,7 +21,8 @@ DataFusion is pulled with all default features, many of which mdvs doesn't use. - `parquet` — read/write parquet files - `sql` — SQL query execution (CREATE VIEW, SELECT, JOIN) -- `nested_expressions` — array functions (`array_has`, `array_length`) used in `--where` +- `nested_expressions` — array functions (`array_has`, `array_length`) used in + `--where` - `string_expressions` — string functions (LIKE, LOWER, UPPER) - `recursive_protection` — stack overflow safety - `unicode_expressions` — possibly needed for Unicode field names @@ -31,7 +33,8 @@ DataFusion is pulled with all default features, many of which mdvs doesn't use. - `crypto_expressions` — md5, sha256 (not used) - `datetime_expressions` — date/time functions (not used, dates are strings) - `encoding_expressions` — base64, hex (not used) -- `compression` — bzip2, xz, flate2, zstd for reading compressed CSV/JSON (not used — Snappy for parquet is built into the parquet crate) +- `compression` — bzip2, xz, flate2, zstd for reading compressed CSV/JSON (not + used — Snappy for parquet is built into the parquet crate) ### Change diff --git a/docs/spec/todos/TODO-0117.md b/docs/spec/todos/TODO-0117.md index 86f7fc8..ff273a1 100644 --- a/docs/spec/todos/TODO-0117.md +++ b/docs/spec/todos/TODO-0117.md @@ -14,31 +14,47 @@ blocks: [] ## Summary -`check_field_values()` in `src/cmd/check.rs` line 401 does `if value.is_null() { continue; }`, which skips ALL subsequent checks for null values. This means null values never trigger `Disallowed` (field present at a wrong path) or `NullNotAllowed` (null on a non-nullable field) unless the field is required. +`check_field_values()` in `src/cmd/check.rs` line 401 does +`if value.is_null() { continue; }`, which skips ALL subsequent checks for null +values. This means null values never trigger `Disallowed` (field present at a +wrong path) or `NullNotAllowed` (null on a non-nullable field) unless the field +is required. ## Problem The four violation checks are independent and orthogonal: -1. **WrongType** — does the value match the declared type? Null always passes (null is accepted by any type). -2. **Disallowed** — is the field present at a path not in `allowed`? The key is there regardless of whether the value is null. +1. **WrongType** — does the value match the declared type? Null always passes + (null is accepted by any type). +2. **Disallowed** — is the field present at a path not in `allowed`? The key is + there regardless of whether the value is null. 3. **NullNotAllowed** — is the value null and `nullable = false`? -4. **MissingRequired** — is the field absent in a required path? (Handled in `check_required_fields()`, works correctly.) +4. **MissingRequired** — is the field absent in a required path? (Handled in + `check_required_fields()`, works correctly.) -All four should run independently on every field in every file. A field can trigger both `Disallowed` AND `NullNotAllowed` simultaneously. +All four should run independently on every field in every file. A field can +trigger both `Disallowed` AND `NullNotAllowed` simultaneously. -Currently, `continue` on null prevents checks 2 and 3 from running. Only `check_required_fields()` catches `NullNotAllowed`, and only for files matching a `required` glob. +Currently, `continue` on null prevents checks 2 and 3 from running. Only +`check_required_fields()` catches `NullNotAllowed`, and only for files matching +a `required` glob. ## Reproduction -In the Refractions vault, `projects` has `nullable = true` inferred because 5 files have bare `projects:` (YAML null). If you change `nullable` to `false` in `mdvs.toml`, `mdvs check` still reports no violations — because those 5 files are in non-required paths (`technologies/rust/concepts/`, etc.) and the null `continue` skips them entirely. +In the Refractions vault, `projects` has `nullable = true` inferred because 5 +files have bare `projects:` (YAML null). If you change `nullable` to `false` in +`mdvs.toml`, `mdvs check` still reports no violations — because those 5 files +are in non-required paths (`technologies/rust/concepts/`, etc.) and the null +`continue` skips them entirely. ## Fix In `check_field_values()`, remove the unconditional `continue` on null. Instead: + - Run `Disallowed` check normally (null or not, the key is present) - Run `NullNotAllowed` check (if field is in toml and `nullable = false`) -- Do NOT run `WrongType` check (null is accepted by any type — no value to compare) +- Do NOT run `WrongType` check (null is accepted by any type — no value to + compare) ## Files diff --git a/docs/spec/todos/TODO-0118.md b/docs/spec/todos/TODO-0118.md index c30b751..e63e7d5 100644 --- a/docs/spec/todos/TODO-0118.md +++ b/docs/spec/todos/TODO-0118.md @@ -13,21 +13,31 @@ blocks: [] ## Summary -The README and book introduction don't surface mdvs's core differentiator: it infers schema constraints from directory structure — different fields, different requirements, different types per directory. This is unique among all competitors and should be immediately visible. +The README and book introduction don't surface mdvs's core differentiator: it +infers schema constraints from directory structure — different fields, different +requirements, different types per directory. This is unique among all +competitors and should be immediately visible. ## Problem -The current README Quick Start uses generic examples (`mdvs init ~/notes`) that don't show path-awareness. The check output (`missing required field 'tags'`) could be any flat linter. Nothing shows "this field isn't allowed in this directory" or "this field is required only in this path." +The current README Quick Start uses generic examples (`mdvs init ~/notes`) that +don't show path-awareness. The check output (`missing required field 'tags'`) +could be any flat linter. Nothing shows "this field isn't allowed in this +directory" or "this field is required only in this path." -The competitive analysis (March 2026) confirmed: no other tool combines schema inference + path-based validation + search. But the README reads like a feature list, not a demonstration. +The competitive analysis (March 2026) confirmed: no other tool combines schema +inference + path-based validation + search. But the README reads like a feature +list, not a demonstration. ## Approach: show, don't tell -Instead of a "Why mdvs?" sales pitch, demonstrate the three dimensions in action with a tiny, self-explanatory example. +Instead of a "Why mdvs?" sales pitch, demonstrate the three dimensions in action +with a tiny, self-explanatory example. ### README -Create a minimal example structure (5-6 files, 3 directories) that makes the point instantly: +Create a minimal example structure (5-6 files, 3 directories) that makes the +point instantly: ``` notes/ @@ -42,15 +52,21 @@ notes/ ``` Three commands, three "aha" moments: -1. **Init** — output shows `draft` only in `blog/`, `role` only in `people/`, `attendees` only in `meetings/`. User sees: "it understood my structure." -2. **Check** — catches a structural mistake (e.g., `draft` leaked into `people/`, or missing `role`). User sees: "it caught something a flat linter wouldn't." + +1. **Init** — output shows `draft` only in `blog/`, `role` only in `people/`, + `attendees` only in `meetings/`. User sees: "it understood my structure." +2. **Check** — catches a structural mistake (e.g., `draft` leaked into + `people/`, or missing `role`). User sees: "it caught something a flat linter + wouldn't." 3. **Search** — finds relevant content. User sees: "it works." -This replaces the current Quick Start section. Keep it tight — the whole example should be scrollable without pausing. +This replaces the current Quick Start section. Keep it tight — the whole example +should be scrollable without pausing. ### Book intro / getting-started The full `example_kb` walkthrough goes deeper: + - Show the complete directory tree - Show the inferred schema with all path patterns - Walk through type widening, nullable, nested objects @@ -59,9 +75,12 @@ The full `example_kb` walkthrough goes deeper: ### Considerations -- The README example should be a real, runnable mini-vault (maybe ship it as `examples/quick-start/` in the repo, or generate it in the Quick Start instructions) +- The README example should be a real, runnable mini-vault (maybe ship it as + `examples/quick-start/` in the repo, or generate it in the Quick Start + instructions) - The output examples must be real captured output, not pseudocode -- The book already has `example_kb` — it needs better framing ("notice how `drift_rate` is required only in `projects/alpha/notes/`"), not new content +- The book already has `example_kb` — it needs better framing ("notice how + `drift_rate` is required only in `projects/alpha/notes/`"), not new content ## Files diff --git a/docs/spec/todos/TODO-0119.md b/docs/spec/todos/TODO-0119.md index a6ec53e..a0a1d91 100644 --- a/docs/spec/todos/TODO-0119.md +++ b/docs/spec/todos/TODO-0119.md @@ -14,21 +14,40 @@ supersedes: [110] ## Summary -Replace the current three-layer output model (`ProcessOutput` structs + `*Result` structs + `*CommandOutput` wrappers) and the `pipeline/` step wrapper modules with a single recursive `Step` tree, generic over the outcome type. Both leaf steps and commands are `Step` nodes. Commands are steps with substeps. +Replace the current three-layer output model (`ProcessOutput` structs + +`*Result` structs + `*CommandOutput` wrappers) and the `pipeline/` step wrapper +modules with a single recursive `Step` tree, generic over the outcome type. +Both leaf steps and commands are `Step` nodes. Commands are steps with substeps. -Verbose/compact is a **struct choice**: `Step` for verbose, `Step` for compact. Both outcome types implement `Render` (produces `Vec`) and `Serialize` (produces JSON). Formatters are shared functions that consume blocks — adding a new output format means writing one function, not touching any command. +Verbose/compact is a **struct choice**: `Step` for verbose, +`Step` for compact. Both outcome types implement `Render` +(produces `Vec`) and `Serialize` (produces JSON). Formatters are shared +functions that consume blocks — adding a new output format means writing one +function, not touching any command. ## Problem -The current architecture has three layers per command (e.g., `BuildProcessOutput` + `BuildResult` + `BuildCommandOutput`), 12 pipeline wrapper modules that add boilerplate around core functions, and per-command manual `has_failed_step()` / `format_text()` implementations that enumerate every field. Adding a step to a command requires touching the ProcessOutput struct, error-handling early returns, format_text branches, and has_failed_step. Build has ~10 early-return blocks each manually constructing the full output struct with `Skipped` for remaining steps. +The current architecture has three layers per command (e.g., +`BuildProcessOutput` + `BuildResult` + `BuildCommandOutput`), 12 pipeline +wrapper modules that add boilerplate around core functions, and per-command +manual `has_failed_step()` / `format_text()` implementations that enumerate +every field. Adding a step to a command requires touching the ProcessOutput +struct, error-handling early returns, format_text branches, and has_failed_step. +Build has ~10 early-return blocks each manually constructing the full output +struct with `Skipped` for remaining steps. -Additionally, per-command `format_text` implementations are full of string-building boilerplate and directly format child structures instead of delegating. Adding a new output format requires writing rendering code in every command. +Additionally, per-command `format_text` implementations are full of +string-building boilerplate and directly format child structures instead of +delegating. Adding a new output format requires writing rendering code in every +command. ## Design ### Core types -`Step` is generic over the outcome type. Commands build `Step` (full data). For compact mode, the tree is converted to `Step` via `to_compact()`. +`Step` is generic over the outcome type. Commands build `Step` (full +data). For compact mode, the tree is converted to `Step` via +`to_compact()`. ```rust struct Step { @@ -74,11 +93,14 @@ impl Step { } ``` -Both `Outcome` and `CompactOutcome` implement `Render + Serialize`. The same struct is used for text rendering AND JSON serialization — no separate paths. +Both `Outcome` and `CompactOutcome` implement `Render + Serialize`. The same +struct is used for text rendering AND JSON serialization — no separate paths. ### Outcome enums -`Outcome` variants use named structs. `CompactOutcome` mirrors the structure with compact counterparts. Every type follows the same pattern: full struct + compact struct + `From` impl. No exceptions. +`Outcome` variants use named structs. `CompactOutcome` mirrors the structure +with compact counterparts. Every type follows the same pattern: full struct + +compact struct + `From` impl. No exceptions. ```rust enum Outcome { @@ -144,11 +166,16 @@ enum CompactOutcome { Command-level outcomes are `Box`ed because they carry `Vec` payloads. -For leaf steps where full and compact are identical, the compact struct can be a type alias or a trivial wrapper with `From` that copies all fields. The pattern is the same regardless — every Outcome variant has a corresponding CompactOutcome variant. +For leaf steps where full and compact are identical, the compact struct can be a +type alias or a trivial wrapper with `From` that copies all fields. The pattern +is the same regardless — every Outcome variant has a corresponding +CompactOutcome variant. ### Compact rendering: leaf steps are silent -In compact mode, **leaf step outcomes return empty vecs** (no blocks). Only command-level outcomes render their summaries and tables. This keeps compact output clean (3-5 lines) matching current behavior. +In compact mode, **leaf step outcomes return empty vecs** (no blocks). Only +command-level outcomes render their summaries and tables. This keeps compact +output clean (3-5 lines) matching current behavior. Example — `mdvs search` with auto-build, compact: @@ -163,6 +190,7 @@ Searched "rust patterns" — 5 hits ``` The tree: + ``` Step(SearchCompact) → renders summary + table ├── Step(BuildCompact) → renders "Built index — 43 files, 142 chunks" @@ -174,7 +202,9 @@ Step(SearchCompact) → renders summary + table └── ... → renders Empty (leaf) ``` -No summary strings needed on parent outcomes — the Build substep's compact outcome renders its own summary line. No data duplication between parent and child. +No summary strings needed on parent outcomes — the Build substep's compact +outcome renders its own summary line. No data duplication between parent and +child. Same command, verbose: @@ -204,11 +234,13 @@ Searched "rust patterns" — 5 hits ### Verbose/compact as struct choice -No `verbose: bool` parameter anywhere in rendering or serialization. The decision happens once at the top level — full tree or compact tree. +No `verbose: bool` parameter anywhere in rendering or serialization. The +decision happens once at the top level — full tree or compact tree. ### Exit code logic -Exit codes are determined on the **full tree** (`Step`), **before** rendering and `to_compact()`. The flow in main.rs: +Exit codes are determined on the **full tree** (`Step`), **before** +rendering and `to_compact()`. The flow in main.rs: ```rust let step: Step = build::run(...).await; @@ -231,24 +263,47 @@ if failed { std::process::exit(2); } if violations { std::process::exit(1); } ``` -`has_failed()` and `has_violations()` are free functions on `Step`, never called on compact trees. `to_compact()` is called only in main.rs dispatch — commands always return full trees. +`has_failed()` and `has_violations()` are free functions on `Step`, +never called on compact trees. `to_compact()` is called only in main.rs dispatch +— commands always return full trees. ### Key principles -1. **Uniform node type**: `Step` — both leaf steps and commands. Leaves have `substeps: vec![]`. -2. **Outcome carries ALL data**: no separate `*Result` structs. Each Outcome variant contains everything needed for its rendering. -3. **Command outcomes are summaries**: aggregate substep data into a denormalized view. Small duplication is intentional. -4. **Verbose/compact is a struct choice**: `Step` vs `Step`. No `verbose` flag in Render or Serialize. -5. **Generic Step**: one type, two instantiations. `to_compact()` converts recursively. -6. **Leaf compact outcomes are silent**: leaf `CompactOutcome` variants return empty vecs (no blocks). Only command-level compact outcomes render summaries and tables. -7. **No summary string duplication**: compact command outcomes don't carry summary strings for nested commands. The nested command's own compact outcome renders its summary line. Data flows through the tree, not through parent fields. -8. **Consistent compact pattern**: every type has a compact counterpart. Full struct + compact struct + `From` impl. No exceptions, even when full and compact are identical. -9. **Block-based rendering**: data types produce `Vec`. Shared formatters convert blocks to text, markdown, etc. -10. **Self-rendering sub-structures**: every data type implements `Render`. Parents compose by collecting child blocks. -11. **Skipped steps are explicit**: remaining steps after failure get `StepOutcome::Skipped`. Steps skipped due to !verbose also get Skipped. -12. **Typed data flows outside the tree**: functional pipeline data (`ScannedFiles`, etc.) lives as local variables in `run()`. Outcome structs carry only aggregated/serializable data — never `ScannedFiles`, `MdvsToml`, or other functional pipeline types. -13. **Errors are structured**: `StepError { kind, message }`. Validation violations are successful outcomes (`Ok`). Build aborting on violations is an error (`Err`). Errors live in `StepOutcome`, never in `Outcome` variants. -14. **Pre-checks are steps**: config mutation, config change detection, dimension mismatch, index metadata read become leaf steps. +1. **Uniform node type**: `Step` — both leaf steps and commands. Leaves have + `substeps: vec![]`. +2. **Outcome carries ALL data**: no separate `*Result` structs. Each Outcome + variant contains everything needed for its rendering. +3. **Command outcomes are summaries**: aggregate substep data into a + denormalized view. Small duplication is intentional. +4. **Verbose/compact is a struct choice**: `Step` vs + `Step`. No `verbose` flag in Render or Serialize. +5. **Generic Step**: one type, two instantiations. `to_compact()` converts + recursively. +6. **Leaf compact outcomes are silent**: leaf `CompactOutcome` variants return + empty vecs (no blocks). Only command-level compact outcomes render summaries + and tables. +7. **No summary string duplication**: compact command outcomes don't carry + summary strings for nested commands. The nested command's own compact outcome + renders its summary line. Data flows through the tree, not through parent + fields. +8. **Consistent compact pattern**: every type has a compact counterpart. Full + struct + compact struct + `From` impl. No exceptions, even when full and + compact are identical. +9. **Block-based rendering**: data types produce `Vec`. Shared formatters + convert blocks to text, markdown, etc. +10. **Self-rendering sub-structures**: every data type implements `Render`. + Parents compose by collecting child blocks. +11. **Skipped steps are explicit**: remaining steps after failure get + `StepOutcome::Skipped`. Steps skipped due to !verbose also get Skipped. +12. **Typed data flows outside the tree**: functional pipeline data + (`ScannedFiles`, etc.) lives as local variables in `run()`. Outcome structs + carry only aggregated/serializable data — never `ScannedFiles`, `MdvsToml`, + or other functional pipeline types. +13. **Errors are structured**: `StepError { kind, message }`. Validation + violations are successful outcomes (`Ok`). Build aborting on violations is + an error (`Err`). Errors live in `StepOutcome`, never in `Outcome` variants. +14. **Pre-checks are steps**: config mutation, config change detection, + dimension mismatch, index metadata read become leaf steps. ### Rendering architecture @@ -283,7 +338,8 @@ enum TableStyle { } ``` -The Render impl (on data types) decides the structure. The formatter handles mechanics (tabled for text, pipe tables for markdown). +The Render impl (on data types) decides the structure. The formatter handles +mechanics (tabled for text, pipe tables for markdown). #### Render trait @@ -296,6 +352,7 @@ trait Render { No parameters. The struct IS the decision. Leaf compact outcomes return empty vec (no blocks): + ```rust impl Render for ScanOutcomeCompact { fn render(&self) -> Vec { vec![] } @@ -303,6 +360,7 @@ impl Render for ScanOutcomeCompact { ``` Full leaf outcomes render one-liners (without timing — Step injects it): + ```rust impl Render for ScanOutcome { fn render(&self) -> Vec { @@ -312,6 +370,7 @@ impl Render for ScanOutcome { ``` Command outcomes render their summaries and tables: + ```rust impl Render for BuildOutcomeCompact { fn render(&self) -> Vec { @@ -321,7 +380,9 @@ impl Render for BuildOutcomeCompact { } ``` -Compact parents collect child compact rows into unified tables where appropriate: +Compact parents collect child compact rows into unified tables where +appropriate: + ```rust impl Render for CheckOutcomeCompact { fn render(&self) -> Vec { @@ -397,7 +458,11 @@ impl Render for StepOutcome { } ``` -**Timing strategy**: `elapsed_ms` lives in `StepOutcome`, not in Outcome structs. Leaf step outcomes render one-liners without timing ("Scan: 43 files"). `Step::render()` injects timing into the first `Block::Line` for leaf steps ("Scan: 43 files (15ms)"). Command outcomes don't get timing injected — their timing is the sum of substeps. +**Timing strategy**: `elapsed_ms` lives in `StepOutcome`, not in Outcome +structs. Leaf step outcomes render one-liners without timing ("Scan: 43 files"). +`Step::render()` injects timing into the first `Block::Line` for leaf steps +("Scan: 43 files (15ms)"). Command outcomes don't get timing injected — their +timing is the sum of substeps. #### Shared formatters @@ -419,7 +484,9 @@ fn format_markdown(blocks: &[Block]) -> String { ### JSON serialization -`Step` hand-implements `Serialize` (does not derive) because `StepOutcome` requires custom serialization to flatten the `Result` nesting. Custom serialization on `StepOutcome`: +`Step` hand-implements `Serialize` (does not derive) because `StepOutcome` +requires custom serialization to flatten the `Result` nesting. Custom +serialization on `StepOutcome`: ```json { "status": "complete", "elapsed_ms": 15, "outcome": { "Scan": { "files_found": 43 } } } @@ -427,14 +494,17 @@ fn format_markdown(blocks: &[Block]) -> String { { "status": "skipped" } ``` -Outcome uses externally tagged serialization (serde default): `{ "Build": { ... } }`. Box is transparent to serde. +Outcome uses externally tagged serialization (serde default): +`{ "Build": { ... } }`. Box is transparent to serde. -Verbose JSON: serialize `Step` — full tree with all substeps and full outcomes. -Compact JSON: serialize `Step` — same tree structure, compact outcomes at each node. +Verbose JSON: serialize `Step` — full tree with all substeps and full +outcomes. Compact JSON: serialize `Step` — same tree structure, +compact outcomes at each node. ### Verbose flag affecting pipeline steps -Sometimes verbose causes extra pipeline work (e.g., reading chunk text from disk in search). This is a **pipeline decision**, not a rendering decision: +Sometimes verbose causes extra pipeline work (e.g., reading chunk text from disk +in search). This is a **pipeline decision**, not a rendering decision: ```rust let chunk_text_step = if verbose { @@ -447,9 +517,12 @@ substeps.push(chunk_text_step); ### Validation violations -Validation finding violations is a **successful outcome** for the validate step. `Outcome::Validate(ValidateOutcome { violations: vec![...] })` is `Ok(...)`. +Validation finding violations is a **successful outcome** for the validate step. +`Outcome::Validate(ValidateOutcome { violations: vec![...] })` is `Ok(...)`. -When build finds violations, it aborts. Build's outcome is `Err(StepError { kind: User, message: "3 violations found" })`. The violation detail is in the Validate substep's successful outcome. +When build finds violations, it aborts. Build's outcome is +`Err(StepError { kind: User, message: "3 violations found" })`. The violation +detail is in the Validate substep's successful outcome. ### Command return type @@ -471,31 +544,38 @@ pub fn run(path: &Path, ...) -> Step { ### Nesting -When search auto-builds, build returns its `Step`. Search includes it as a substep. The build step renders identically whether standalone or nested inside search — no context metadata needed. +When search auto-builds, build returns its `Step`. Search includes it +as a substep. The build step renders identically whether standalone or nested +inside search — no context metadata needed. ### What gets deleted - All `pipeline/*.rs` step wrapper modules and their `*Output` structs - All per-command `*ProcessOutput`, `*CommandOutput`, `*Result` structs - `StepOutput` trait, `CommandOutput` trait -- Per-command `has_failed_step()`, `format_text()`, `format_json()` implementations +- Per-command `has_failed_step()`, `format_text()`, `format_json()` + implementations - `format_json_compact` helper ### What gets added - `Step`, `StepOutcome`, `StepError` types (generic) - `Outcome` enum with named outcome structs per step/command -- `CompactOutcome` enum with compact counterpart structs (consistent pattern for all variants) +- `CompactOutcome` enum with compact counterpart structs (consistent pattern for + all variants) - `Block` enum, `TableStyle` enum, `Render` trait -- `Render` implementations on all outcome structs (full and compact) and sub-structures +- `Render` implementations on all outcome structs (full and compact) and + sub-structures - Shared formatters: `format_text()`, `format_markdown()` - Recursive `has_failed`, `has_violations` on `Step` - `#[cfg(test)]` convenience methods for test ergonomics ### What stays -- Core functions (`ScannedFiles::scan()`, `check::validate()`, `classify_files()`, etc.) -- Shared sub-types (`FieldViolation`, `SearchHit`, `BuildFileDetail`, `DiscoveredField`, etc.) — now with compact counterparts and `Render` impls +- Core functions (`ScannedFiles::scan()`, `check::validate()`, + `classify_files()`, etc.) +- Shared sub-types (`FieldViolation`, `SearchHit`, `BuildFileDetail`, + `DiscoveredField`, etc.) — now with compact counterparts and `Render` impls - Table style helpers in `src/table.rs` (used by `format_text` formatter) ### Test helpers @@ -546,38 +626,57 @@ impl Outcome { ## Interaction with other TODOs - **Supersedes TODO-0110**: same problem, different solution. -- **Blocks TODO-0100**: text output redesign becomes about designing the specific blocks each command produces. +- **Blocks TODO-0100**: text output redesign becomes about designing the + specific blocks each command produces. - **Blocks TODO-0101**: markdown format = one new formatter function. - **Affects TODO-0088**: exit code logic via recursive Step tree inspection. ## Follow-up TODOs (to be created after implementation) -- **Macro for compact struct generation**: use `crabtime` to auto-generate `*Compact` structs and `From` impls from annotated full structs. Deferred until the pattern is stable. -- **Macro for step pipeline boilerplate**: the early-return pattern in `run()` (call step → push substep → check error → push Skipped for remaining → return) repeats ~10 times in build. A declarative macro could reduce each step to one line. Deferred until the manual pattern is proven stable. +- **Macro for compact struct generation**: use `crabtime` to auto-generate + `*Compact` structs and `From` impls from annotated full structs. Deferred + until the pattern is stable. +- **Macro for step pipeline boilerplate**: the early-return pattern in `run()` + (call step → push substep → check error → push Skipped for remaining → return) + repeats ~10 times in build. A declarative macro could reduce each step to one + line. Deferred until the manual pattern is proven stable. ## Resolved questions -- **Leaf step timing**: `Step::render()` injects `elapsed_ms` into the first `Block::Line` of leaf steps via `StepOutcome::elapsed_ms()` accessor. Outcome structs don't carry timing. The Render trait stays parameterless. -- **Block::Empty**: removed from the enum. Leaf compact outcomes return `vec![]` (empty vec). Parents extend with empty vec — no-op. No placeholder block needed. -- **Serde on Step**: `Step` hand-implements `Serialize` (not derive) because `StepOutcome` has custom serialization. The custom impl delegates field-by-field. -- **Skipped step boilerplate**: same volume as today, different shape. Accept for now, macro deferred as follow-up TODO. +- **Leaf step timing**: `Step::render()` injects `elapsed_ms` into the first + `Block::Line` of leaf steps via `StepOutcome::elapsed_ms()` accessor. Outcome + structs don't carry timing. The Render trait stays parameterless. +- **Block::Empty**: removed from the enum. Leaf compact outcomes return `vec![]` + (empty vec). Parents extend with empty vec — no-op. No placeholder block + needed. +- **Serde on Step**: `Step` hand-implements `Serialize` (not derive) because + `StepOutcome` has custom serialization. The custom impl delegates + field-by-field. +- **Skipped step boilerplate**: same volume as today, different shape. Accept + for now, macro deferred as follow-up TODO. ## Migration path Convert in reverse dependency order (leaves first): -1. **Infrastructure**: `Step`, `StepOutcome`, `Outcome`, `CompactOutcome`, `Block`, `Render`, formatters -2. **Independent commands**: clean, info, check, init (no command-to-command deps) +1. **Infrastructure**: `Step`, `StepOutcome`, `Outcome`, `CompactOutcome`, + `Block`, `Render`, formatters +2. **Independent commands**: clean, info, check, init (no command-to-command + deps) 3. **update** (called by build) -4. **build + search together** (tight coupling via auto-build/auto-update nesting) +4. **build + search together** (tight coupling via auto-build/auto-update + nesting) ## Files affected - `src/step.rs` (new) — `Step`, `StepOutcome`, `StepError` -- `src/outcome/` (new directory) — `Outcome`, `CompactOutcome` enums + all named outcome structs + compact counterparts +- `src/outcome/` (new directory) — `Outcome`, `CompactOutcome` enums + all named + outcome structs + compact counterparts - `src/block.rs` (new) — `Block`, `TableStyle`, `Render` trait - `src/render.rs` (new) — shared formatters (`format_text`, `format_markdown`) - `src/pipeline/*.rs` — delete all step wrapper modules - `src/cmd/*.rs` — rewrite to build `Step` trees -- `src/output.rs` — simplify (remove `CommandOutput` trait; shared sub-types get compact counterparts) -- `src/main.rs` — dispatch via `Step` + exit code logic + `to_compact()` choice +- `src/output.rs` — simplify (remove `CommandOutput` trait; shared sub-types get + compact counterparts) +- `src/main.rs` — dispatch via `Step` + exit code logic + + `to_compact()` choice diff --git a/docs/spec/todos/TODO-0120.md b/docs/spec/todos/TODO-0120.md index 9188a4c..451a35e 100644 --- a/docs/spec/todos/TODO-0120.md +++ b/docs/spec/todos/TODO-0120.md @@ -15,7 +15,8 @@ files_updated: [src/lib.rs] ## Summary -Create the foundational generic `Step` type and its supporting types. This is the first piece of the TODO-0119 architecture — everything else builds on it. +Create the foundational generic `Step` type and its supporting types. This is +the first piece of the TODO-0119 architecture — everything else builds on it. ## Details @@ -23,21 +24,27 @@ Create `src/step.rs` with: ### Types -- `Step` — generic tree node with `substeps: Vec>` and `outcome: StepOutcome` -- `StepOutcome` — enum: `Complete { result: Result, elapsed_ms: u64 }` | `Skipped` +- `Step` — generic tree node with `substeps: Vec>` and + `outcome: StepOutcome` +- `StepOutcome` — enum: + `Complete { result: Result, elapsed_ms: u64 }` | `Skipped` - `StepError` — struct: `kind: ErrorKind` + `message: String` - `ErrorKind` — enum: `User` | `Application` -- Type aliases: `type FullStep = Step`, `type CompactStep = Step` +- Type aliases: `type FullStep = Step`, + `type CompactStep = Step` ### Methods - `StepOutcome::elapsed_ms(&self) -> Option` — accessor -- `Step::to_compact(&self) -> Step` — recursive conversion (calls `outcome.to_compact(&self.substeps)`) +- `Step::to_compact(&self) -> Step` — recursive + conversion (calls `outcome.to_compact(&self.substeps)`) ### Free functions -- `has_failed(step: &Step) -> bool` — recursive, checks for `Err` in any node -- `has_violations(step: &Step) -> bool` — recursive, delegates to `Outcome::contains_violations()` +- `has_failed(step: &Step) -> bool` — recursive, checks for `Err` in + any node +- `has_violations(step: &Step) -> bool` — recursive, delegates to + `Outcome::contains_violations()` ### Test helpers (`#[cfg(test)]`) @@ -54,7 +61,8 @@ Create `src/step.rs` with: ### Notes -- `to_compact()` requires `Outcome` and `CompactOutcome` to exist (TODO-0122), but the method signature and stub can be written first. +- `to_compact()` requires `Outcome` and `CompactOutcome` to exist (TODO-0122), + but the method signature and stub can be written first. - `StepError` must `derive(Clone)` for the `Err` branch of `to_compact()`. - `Step` does NOT derive `Serialize` — hand-impl comes in TODO-0124. diff --git a/docs/spec/todos/TODO-0121.md b/docs/spec/todos/TODO-0121.md index fc228e2..c75feee 100644 --- a/docs/spec/todos/TODO-0121.md +++ b/docs/spec/todos/TODO-0121.md @@ -15,7 +15,8 @@ files_updated: [src/step.rs, src/lib.rs] ## Summary -Create the rendering primitive types and the `Render` trait. This is the abstraction layer between data and formatters. +Create the rendering primitive types and the `Render` trait. This is the +abstraction layer between data and formatters. ## Details @@ -23,7 +24,8 @@ Create `src/block.rs` with: ### Types -- `Block` enum: `Line(String)` | `Table { headers, rows, style }` | `Section { label, children }` +- `Block` enum: `Line(String)` | `Table { headers, rows, style }` | + `Section { label, children }` - `TableStyle` enum: `Compact` | `Record { detail_rows: Vec }` ### Render trait @@ -38,12 +40,17 @@ No parameters. The struct IS the verbose/compact decision. ### Render impls on Step types -- `impl Render for Step` — recursive: render substeps, then render own outcome. For leaf steps (no substeps), inject `elapsed_ms` into the first `Block::Line`. -- `impl Render for StepOutcome` — match on Complete(Ok) → delegate to outcome, Complete(Err) → error line, Skipped → empty vec. +- `impl Render for Step` — recursive: render substeps, then render + own outcome. For leaf steps (no substeps), inject `elapsed_ms` into the first + `Block::Line`. +- `impl Render for StepOutcome` — match on Complete(Ok) → delegate + to outcome, Complete(Err) → error line, Skipped → empty vec. ### Notes -- The timing injection logic: only leaf steps (substeps.is_empty()) get elapsed_ms appended to their first Block::Line. Command outcomes don't get timing. +- The timing injection logic: only leaf steps (substeps.is_empty()) get + elapsed_ms appended to their first Block::Line. Command outcomes don't get + timing. - Block derives `Debug`, `Clone`. - TableStyle derives `Debug`, `Clone`. diff --git a/docs/spec/todos/TODO-0122.md b/docs/spec/todos/TODO-0122.md index f66f862..57ccb7a 100644 --- a/docs/spec/todos/TODO-0122.md +++ b/docs/spec/todos/TODO-0122.md @@ -13,19 +13,26 @@ blocks: [124, 125, 126, 127, 128, 129, 130, 131] ## Summary -Create the `Outcome` and `CompactOutcome` enums with all named outcome structs, compact counterparts, `From` impls, and `Render` impls. Built incrementally — add variants as each command is converted. +Create the `Outcome` and `CompactOutcome` enums with all named outcome structs, +compact counterparts, `From` impls, and `Render` impls. Built incrementally — +add variants as each command is converted. ## Incremental checklist -The enums start with a `_ => todo!()` catch-all arm in `to_compact()` and `contains_violations()`. Variants are added as commands are converted. Each checkbox corresponds to a command conversion TODO. +The enums start with a `_ => todo!()` catch-all arm in `to_compact()` and +`contains_violations()`. Variants are added as commands are converted. Each +checkbox corresponds to a command conversion TODO. ### Phase 1: minimal (for clean — TODO-0125) -- [x] Create `src/outcome/mod.rs` with `Outcome` and `CompactOutcome` enums (initially: `DeleteIndex` + `Clean` variants only) + +- [x] Create `src/outcome/mod.rs` with `Outcome` and `CompactOutcome` enums + (initially: `DeleteIndex` + `Clean` variants only) - [x] `Outcome::to_compact()` and `Outcome::contains_violations()` - [x] `DeleteIndexOutcome` / `DeleteIndexOutcomeCompact` + `From` + `Render` - [x] `CleanOutcome` / `CleanOutcomeCompact` + `From` + `Render` ### Phase 2: info (TODO-0126) + - [x] Add `ReadConfig`, `Scan`, `ReadIndex`, `Info` variants to both enums - [x] `ReadConfigOutcome` / `ReadConfigOutcomeCompact` + `From` + `Render` - [x] `ScanOutcome` / `ScanOutcomeCompact` + `From` + `Render` @@ -33,6 +40,7 @@ The enums start with a `_ => todo!()` catch-all arm in `to_compact()` and `conta - [x] `InfoOutcome` / `InfoOutcomeCompact` + `From` + `Render` ### Phase 3: check (TODO-0127) + - [x] Add `Validate`, `Check` variants to both enums - [x] `ValidateOutcome` / `ValidateOutcomeCompact` + `From` + `Render` - [x] `CheckOutcome` / `CheckOutcomeCompact` + `From` + `Render` @@ -40,6 +48,7 @@ The enums start with a `_ => todo!()` catch-all arm in `to_compact()` and `conta - [x] Update `Outcome::contains_violations()` for Validate and Check variants ### Phase 4: init (TODO-0128) + - [x] Add `Infer`, `WriteConfig`, `Init` variants to both enums - [x] `InferOutcome` / `InferOutcomeCompact` + `From` + `Render` - [x] `WriteConfigOutcome` / `WriteConfigOutcomeCompact` + `From` + `Render` @@ -47,22 +56,29 @@ The enums start with a `_ => todo!()` catch-all arm in `to_compact()` and `conta - [x] `DiscoveredFieldCompact` sub-structure compact + `From` ### Phase 5: update (TODO-0129) + - [x] Add `Update` variant to both enums - [x] `UpdateOutcome` / `UpdateOutcomeCompact` + `From` + `Render` -- [x] `ChangedFieldCompact`, `RemovedFieldCompact` sub-structure compacts + `From` +- [x] `ChangedFieldCompact`, `RemovedFieldCompact` sub-structure compacts + + `From` ### Phase 6: build + search (TODO-0130) -- [ ] Add `MutateConfig`, `ReadIndexMetadata`, `CheckConfigChanged`, `Classify`, `LoadModel`, `EmbedFiles`, `WriteIndex`, `Build` variants to both enums -- [ ] Add `EmbedQuery`, `ExecuteSearch`, `ReadChunkText`, `Search` variants to both enums + +- [ ] Add `MutateConfig`, `ReadIndexMetadata`, `CheckConfigChanged`, `Classify`, + `LoadModel`, `EmbedFiles`, `WriteIndex`, `Build` variants to both enums +- [ ] Add `EmbedQuery`, `ExecuteSearch`, `ReadChunkText`, `Search` variants to + both enums - [ ] All remaining outcome structs + compact counterparts + `From` + `Render` -- [ ] `SearchHitCompact`, `BuildFileDetailCompact` sub-structure compacts + `From` +- [ ] `SearchHitCompact`, `BuildFileDetailCompact` sub-structure compacts + + `From` - [ ] Remove `_ => todo!()` catch-all arms — all variants now covered ## Details ### Outcome structs (full + compact pairs) -Each pair: `derive(Debug, Serialize)` on both, `From<&Full> for Compact` impl. ~34 `From` impls total (24 outcome pairs + ~10 sub-structure pairs). +Each pair: `derive(Debug, Serialize)` on both, `From<&Full> for Compact` impl. +~34 `From` impls total (24 outcome pairs + ~10 sub-structure pairs). ### Sub-structure compact counterparts @@ -83,26 +99,35 @@ Each pair: `derive(Debug, Serialize)` on both, `From<&Full> for Compact` impl. ~ ### Key methods -- `Outcome::to_compact(&self, substeps: &[Step]) -> CompactOutcome` — match with `_ => todo!()` initially, filled incrementally -- `Outcome::contains_violations(&self) -> bool` — for exit code logic, filled incrementally +- `Outcome::to_compact(&self, substeps: &[Step]) -> CompactOutcome` — + match with `_ => todo!()` initially, filled incrementally +- `Outcome::contains_violations(&self) -> bool` — for exit code logic, filled + incrementally ### Notes -- Outcome structs carry only aggregated/serializable data — never ScannedFiles, MdvsToml, or other functional pipeline types. -- Outcome and CompactOutcome derive `Serialize` with externally tagged (serde default). -- Command outcomes may need to read substep data during `to_compact()` for summaries. +- Outcome structs carry only aggregated/serializable data — never ScannedFiles, + MdvsToml, or other functional pipeline types. +- Outcome and CompactOutcome derive `Serialize` with externally tagged (serde + default). +- Command outcomes may need to read substep data during `to_compact()` for + summaries. ## Files -- `src/outcome/mod.rs` (new) — Outcome, CompactOutcome enums, to_compact(), contains_violations() +- `src/outcome/mod.rs` (new) — Outcome, CompactOutcome enums, to_compact(), + contains_violations() - `src/outcome/scan.rs` (new) — ScanOutcome, ScanOutcomeCompact, Render impls - `src/outcome/infer.rs` (new) -- `src/outcome/config.rs` (new) — ReadConfig, WriteConfig, MutateConfig, CheckConfigChanged +- `src/outcome/config.rs` (new) — ReadConfig, WriteConfig, MutateConfig, + CheckConfigChanged - `src/outcome/validate.rs` (new) - `src/outcome/classify.rs` (new) - `src/outcome/model.rs` (new) — LoadModel - `src/outcome/embed.rs` (new) — EmbedFiles, EmbedQuery -- `src/outcome/index.rs` (new) — ReadIndex, ReadIndexMetadata, WriteIndex, DeleteIndex +- `src/outcome/index.rs` (new) — ReadIndex, ReadIndexMetadata, WriteIndex, + DeleteIndex - `src/outcome/search.rs` (new) — ExecuteSearch, ReadChunkText -- `src/outcome/commands.rs` (new) — Init, Update, Check, Build, Search, Info, Clean outcomes +- `src/outcome/commands.rs` (new) — Init, Update, Check, Build, Search, Info, + Clean outcomes - `src/lib.rs` (add `pub mod outcome`) diff --git a/docs/spec/todos/TODO-0123.md b/docs/spec/todos/TODO-0123.md index 4e93c92..769f46b 100644 --- a/docs/spec/todos/TODO-0123.md +++ b/docs/spec/todos/TODO-0123.md @@ -15,7 +15,9 @@ files_updated: [src/lib.rs] ## Summary -Create the shared formatter functions that consume `Vec` and produce formatted output strings. These are the "how to show" layer — commands never contain format-specific code. +Create the shared formatter functions that consume `Vec` and produce +formatted output strings. These are the "how to show" layer — commands never +contain format-specific code. ## Details @@ -24,22 +26,28 @@ Create `src/render.rs` with: ### format_text(blocks: &[Block]) -> String - `Block::Line(s)` → print as-is with newline -- `Block::Table { style: Compact, .. }` → tabled crate with `style_compact()` from `src/table.rs` -- `Block::Table { style: Record { detail_rows }, .. }` → tabled crate with `style_record()` + ColumnSpan on specified detail row indices -- `Block::Section { label, children }` → print label, indent each child block by 2 spaces +- `Block::Table { style: Compact, .. }` → tabled crate with `style_compact()` + from `src/table.rs` +- `Block::Table { style: Record { detail_rows }, .. }` → tabled crate with + `style_record()` + ColumnSpan on specified detail row indices +- `Block::Section { label, children }` → print label, indent each child block by + 2 spaces ### format_markdown(blocks: &[Block]) -> String - `Block::Line(s)` → print as-is with newline -- `Block::Table { .. }` → standard markdown pipe table with `|` columns and `---` header separators +- `Block::Table { .. }` → standard markdown pipe table with `|` columns and + `---` header separators - `Block::Section { label, children }` → `## label` header, then render children ### Notes -- Both formatters use the existing `style_compact()` and `style_record()` helpers from `src/table.rs`. +- Both formatters use the existing `style_compact()` and `style_record()` + helpers from `src/table.rs`. - `format_markdown` is needed for TODO-0101 but can be stubbed initially. - Formatters must handle empty block vecs gracefully (return empty string). -- Section indentation: each line of each child block's rendered text gets 2-space indent. +- Section indentation: each line of each child block's rendered text gets + 2-space indent. ## Files diff --git a/docs/spec/todos/TODO-0124.md b/docs/spec/todos/TODO-0124.md index 9b57f74..de858d1 100644 --- a/docs/spec/todos/TODO-0124.md +++ b/docs/spec/todos/TODO-0124.md @@ -14,7 +14,9 @@ files_updated: [src/step.rs] ## Summary -Hand-implement `Serialize` for `Step` and `StepOutcome` to produce clean JSON output. Done once — works for all Outcome variants as they're added incrementally in TODO-0122. +Hand-implement `Serialize` for `Step` and `StepOutcome` to produce clean +JSON output. Done once — works for all Outcome variants as they're added +incrementally in TODO-0122. ## Details @@ -28,7 +30,8 @@ Flatten `Result` into a discriminated object: { "status": "skipped" } ``` -Implementation uses `serialize_struct` with conditional fields based on the variant. +Implementation uses `serialize_struct` with conditional fields based on the +variant. ### Step custom Serialize @@ -45,16 +48,22 @@ impl Serialize for Step { } ``` -Works recursively: `Vec>` serializes each element via the same hand-impl. +Works recursively: `Vec>` serializes each element via the same +hand-impl. ### Outcome and CompactOutcome -These use `derive(Serialize)` with externally tagged (serde default). Each variant serializes as `{ "VariantName": { fields } }`. Box is transparent to serde. No work needed here — the derive handles it as variants are added in TODO-0122. +These use `derive(Serialize)` with externally tagged (serde default). Each +variant serializes as `{ "VariantName": { fields } }`. Box is transparent to +serde. No work needed here — the derive handles it as variants are added in +TODO-0122. ### Notes -- This TODO can be implemented as soon as TODO-0120 and the initial Outcome enum exist (even with just 2 variants). -- Once done, it never needs updating — the generic impl works for any `O: Serialize`. +- This TODO can be implemented as soon as TODO-0120 and the initial Outcome enum + exist (even with just 2 variants). +- Once done, it never needs updating — the generic impl works for any + `O: Serialize`. - `StepError` derives `Serialize` normally. - `ErrorKind` derives `Serialize` with `#[serde(rename_all = "snake_case")]`. diff --git a/docs/spec/todos/TODO-0125.md b/docs/spec/todos/TODO-0125.md index 2ee796c..7a41d82 100644 --- a/docs/spec/todos/TODO-0125.md +++ b/docs/spec/todos/TODO-0125.md @@ -14,32 +14,45 @@ files_updated: [src/cmd/clean.rs, src/main.rs, src/step.rs] ## Summary -Convert the `clean` command to the new Step tree architecture. Simplest command — one leaf step (delete_index). First end-to-end test of the full pipeline. +Convert the `clean` command to the new Step tree architecture. Simplest command +— one leaf step (delete_index). First end-to-end test of the full pipeline. ## Checklist ### Outcome structs (part of TODO-0122 phase 1) -- [ ] Add `DeleteIndex` + `Clean` variants to `Outcome` and `CompactOutcome` enums -- [ ] Create `DeleteIndexOutcome` / `DeleteIndexOutcomeCompact` + `From` + `Render` + +- [ ] Add `DeleteIndex` + `Clean` variants to `Outcome` and `CompactOutcome` + enums +- [ ] Create `DeleteIndexOutcome` / `DeleteIndexOutcomeCompact` + `From` + + `Render` - [ ] Create `CleanOutcome` / `CleanOutcomeCompact` + `From` + `Render` -- [ ] Implement `Outcome::to_compact()` and `contains_violations()` with `_ => todo!()` catch-all +- [ ] Implement `Outcome::to_compact()` and `contains_violations()` with + `_ => todo!()` catch-all ### Command rewrite + - [ ] Rewrite `clean::run()` → returns `Step` - [ ] Delete `CleanProcessOutput`, `CleanCommandOutput`, `CleanResult` structs - [ ] Delete `CommandOutput` impl on `CleanResult` and `CleanCommandOutput` - [ ] Delete `has_failed_step()` on `CleanCommandOutput` ### main.rs -- [ ] Update clean match arm: `Step` dispatch with exit codes, `to_compact()`, formatter + +- [ ] Update clean match arm: `Step` dispatch with exit codes, + `to_compact()`, formatter ### Tests -- [ ] Rewrite clean tests with `#[cfg(test)]` helpers (`unwrap_clean()`, `has_failed()`, `is_skipped()`) + +- [ ] Rewrite clean tests with `#[cfg(test)]` helpers (`unwrap_clean()`, + `has_failed()`, `is_skipped()`) ### Verification -- [ ] `cargo test` — all tests pass (clean tests + other commands still using old model) + +- [ ] `cargo test` — all tests pass (clean tests + other commands still using + old model) - [ ] `cargo clippy` + `cargo fmt` -- [ ] Manual: `cargo run -- clean example_kb` — text compact, text verbose, JSON compact, JSON verbose all correct +- [ ] Manual: `cargo run -- clean example_kb` — text compact, text verbose, JSON + compact, JSON verbose all correct ## Files diff --git a/docs/spec/todos/TODO-0126.md b/docs/spec/todos/TODO-0126.md index e21e913..0f5db07 100644 --- a/docs/spec/todos/TODO-0126.md +++ b/docs/spec/todos/TODO-0126.md @@ -15,29 +15,37 @@ files_updated: [src/outcome/index.rs, src/outcome/commands.rs, src/outcome/mod.r ## Summary -Convert the `info` command to the new Step tree architecture. Simple query/display command with 2-3 leaf steps. +Convert the `info` command to the new Step tree architecture. Simple +query/display command with 2-3 leaf steps. ## Checklist ### Outcome structs (part of TODO-0122 phase 2) -- [ ] Add `ReadConfig`, `Scan`, `ReadIndex`, `Info` variants to `Outcome` and `CompactOutcome` enums -- [ ] Create `ReadConfigOutcome` / `ReadConfigOutcomeCompact` + `From` + `Render` + +- [ ] Add `ReadConfig`, `Scan`, `ReadIndex`, `Info` variants to `Outcome` and + `CompactOutcome` enums +- [ ] Create `ReadConfigOutcome` / `ReadConfigOutcomeCompact` + `From` + + `Render` - [ ] Create `ScanOutcome` / `ScanOutcomeCompact` + `From` + `Render` - [ ] Create `ReadIndexOutcome` / `ReadIndexOutcomeCompact` + `From` + `Render` - [ ] Create `InfoOutcome` / `InfoOutcomeCompact` + `From` + `Render` ### Command rewrite + - [ ] Rewrite `info::run()` → returns `Step` - [ ] Delete `InfoProcessOutput`, `InfoCommandOutput`, `InfoResult` structs - [ ] Delete `CommandOutput` impls ### main.rs + - [ ] Update info match arm: `Step` dispatch ### Tests + - [ ] Rewrite info tests with `#[cfg(test)]` helpers ### Verification + - [ ] `cargo test` — all tests pass - [ ] `cargo clippy` + `cargo fmt` - [ ] Manual: `cargo run -- info example_kb` — all output formats correct diff --git a/docs/spec/todos/TODO-0127.md b/docs/spec/todos/TODO-0127.md index 4e0be91..d2273e0 100644 --- a/docs/spec/todos/TODO-0127.md +++ b/docs/spec/todos/TODO-0127.md @@ -15,11 +15,13 @@ files_updated: [src/outcome/commands.rs, src/outcome/mod.rs, src/output.rs, src/ ## Summary -Convert the `check` command to the new Step tree architecture. Medium complexity — has violations rendering, new fields, and optional auto-update nesting. +Convert the `check` command to the new Step tree architecture. Medium complexity +— has violations rendering, new fields, and optional auto-update nesting. ## Checklist ### Outcome structs (part of TODO-0122 phase 3) + - [ ] Add `Validate`, `Check` variants to `Outcome` and `CompactOutcome` enums - [ ] Create `ValidateOutcome` / `ValidateOutcomeCompact` + `From` + `Render` - [ ] Create `CheckOutcome` / `CheckOutcomeCompact` + `From` + `Render` @@ -28,27 +30,34 @@ Convert the `check` command to the new Step tree architecture. Medium complexity - [ ] Update `Outcome::contains_violations()` for Validate and Check variants ### Command rewrite + - [ ] Rewrite `check::run()` → returns `Step` - [ ] Keep `check::validate()` as core function (reusable by build) - [ ] Delete `CheckProcessOutput`, `CheckCommandOutput`, `CheckResult` structs - [ ] Delete `CommandOutput` impls -- [ ] Handle auto-update nesting: if update is already converted, nest its Step; otherwise stub +- [ ] Handle auto-update nesting: if update is already converted, nest its Step; + otherwise stub ### main.rs + - [ ] Update check match arm: `Step` dispatch ### Tests -- [ ] Rewrite check tests with `#[cfg(test)]` helpers (`unwrap_check()`, `has_failed()`) + +- [ ] Rewrite check tests with `#[cfg(test)]` helpers (`unwrap_check()`, + `has_failed()`) - [ ] Test violation detection, new fields, auto-update ### Verification + - [ ] `cargo test` — all tests pass - [ ] `cargo clippy` + `cargo fmt` - [ ] Manual: `cargo run -- check example_kb` — all output formats correct ## Files -- `src/cmd/check.rs` (rewrite run(), keep validate(), delete old structs, rewrite tests) +- `src/cmd/check.rs` (rewrite run(), keep validate(), delete old structs, + rewrite tests) - `src/outcome/` (add Validate, Check variants and structs) - `src/output.rs` (add FieldViolationCompact, NewFieldCompact) - `src/main.rs` (update check match arm) diff --git a/docs/spec/todos/TODO-0128.md b/docs/spec/todos/TODO-0128.md index 34c23e6..ca53f3b 100644 --- a/docs/spec/todos/TODO-0128.md +++ b/docs/spec/todos/TODO-0128.md @@ -15,29 +15,38 @@ files_updated: [src/outcome/config.rs, src/outcome/commands/mod.rs, src/outcome/ ## Summary -Convert the `init` command to the new Step tree architecture. Simple pipeline — scan, infer, write_config. +Convert the `init` command to the new Step tree architecture. Simple pipeline — +scan, infer, write_config. ## Checklist ### Outcome structs (part of TODO-0122 phase 4) -- [ ] Add `Infer`, `WriteConfig`, `Init` variants to `Outcome` and `CompactOutcome` enums + +- [ ] Add `Infer`, `WriteConfig`, `Init` variants to `Outcome` and + `CompactOutcome` enums - [ ] Create `InferOutcome` / `InferOutcomeCompact` + `From` + `Render` -- [ ] Create `WriteConfigOutcome` / `WriteConfigOutcomeCompact` + `From` + `Render` +- [ ] Create `WriteConfigOutcome` / `WriteConfigOutcomeCompact` + `From` + + `Render` - [ ] Create `InitOutcome` / `InitOutcomeCompact` + `From` + `Render` - [ ] Create `DiscoveredFieldCompact` + `From<&DiscoveredField>` ### Command rewrite + - [ ] Rewrite `init::run()` → returns `Step` - [ ] Delete `InitProcessOutput`, `InitCommandOutput`, `InitResult` structs - [ ] Delete `CommandOutput` impls ### main.rs + - [ ] Update init match arm: `Step` dispatch ### Tests -- [ ] Rewrite init tests with `#[cfg(test)]` helpers (`unwrap_init()`, `has_failed()`) + +- [ ] Rewrite init tests with `#[cfg(test)]` helpers (`unwrap_init()`, + `has_failed()`) ### Verification + - [ ] `cargo test` — all tests pass - [ ] `cargo clippy` + `cargo fmt` - [ ] Manual: `cargo run -- init /tmp/test-vault` — all output formats correct diff --git a/docs/spec/todos/TODO-0129.md b/docs/spec/todos/TODO-0129.md index d46f826..0b786ec 100644 --- a/docs/spec/todos/TODO-0129.md +++ b/docs/spec/todos/TODO-0129.md @@ -15,30 +15,38 @@ files_updated: [src/outcome/commands/mod.rs, src/outcome/mod.rs, src/output.rs, ## Summary -Convert the `update` command to the new Step tree architecture. Must be done before build (which nests update as auto-update). +Convert the `update` command to the new Step tree architecture. Must be done +before build (which nests update as auto-update). ## Checklist ### Outcome structs (part of TODO-0122 phase 5) + - [ ] Add `Update` variant to `Outcome` and `CompactOutcome` enums - [ ] Create `UpdateOutcome` / `UpdateOutcomeCompact` + `From` + `Render` - [ ] Create `ChangedFieldCompact` + `From<&ChangedField>` - [ ] Create `RemovedFieldCompact` + `From<&RemovedField>` ### Command rewrite + - [ ] Rewrite `update::run()` → returns `Step` - [ ] Field comparison logic stays inline in `run()` -- [ ] Delete `UpdateProcessOutput`, `UpdateCommandOutput`, `UpdateResult` structs +- [ ] Delete `UpdateProcessOutput`, `UpdateCommandOutput`, `UpdateResult` + structs - [ ] Delete `CommandOutput` impls ### main.rs + - [ ] Update update match arm: `Step` dispatch ### Tests -- [ ] Rewrite update tests with `#[cfg(test)]` helpers (`unwrap_update()`, `has_failed()`) + +- [ ] Rewrite update tests with `#[cfg(test)]` helpers (`unwrap_update()`, + `has_failed()`) - [ ] Test reinfer, reinfer-all, dry-run, new fields, changed fields ### Verification + - [ ] `cargo test` — all tests pass - [ ] `cargo clippy` + `cargo fmt` - [ ] Manual: `cargo run -- update example_kb` — all output formats correct diff --git a/docs/spec/todos/TODO-0130.md b/docs/spec/todos/TODO-0130.md index ef17207..cc010ce 100644 --- a/docs/spec/todos/TODO-0130.md +++ b/docs/spec/todos/TODO-0130.md @@ -12,33 +12,47 @@ blocks: [131] ## Summary -Convert `build` and `search` commands to the new Step tree architecture. Split into two waves: build first (nests update, already converted), then search (nests build). +Convert `build` and `search` commands to the new Step tree architecture. Split +into two waves: build first (nests update, already converted), then search +(nests build). ## Wave 1: Build command ### Outcome structs (part of TODO-0122 phase 6) -- [x] Add `Classify`, `LoadModel`, `EmbedFiles`, `WriteIndex`, `Build` variants to both enums + +- [x] Add `Classify`, `LoadModel`, `EmbedFiles`, `WriteIndex`, `Build` variants + to both enums - [x] Create `ClassifyOutcome` / `ClassifyOutcomeCompact` + `From` + `Render` - [x] Create `LoadModelOutcome` / `LoadModelOutcomeCompact` + `From` + `Render` -- [x] Create `EmbedFilesOutcome` / `EmbedFilesOutcomeCompact` + `From` + `Render` -- [x] Create `WriteIndexOutcome` / `WriteIndexOutcomeCompact` + `From` + `Render` +- [x] Create `EmbedFilesOutcome` / `EmbedFilesOutcomeCompact` + `From` + + `Render` +- [x] Create `WriteIndexOutcome` / `WriteIndexOutcomeCompact` + `From` + + `Render` - [x] Create `BuildOutcome` / `BuildOutcomeCompact` + `From` + `Render` -- Skipped: MutateConfig, ReadIndexMetadata, CheckConfigChanged, BuildFileDetailCompact — pre-checks stay inline +- Skipped: MutateConfig, ReadIndexMetadata, CheckConfigChanged, + BuildFileDetailCompact — pre-checks stay inline ### Build command rewrite + - [x] Rewrite `build::run()` → returns `Step` -- [x] Pre-checks stay inline (MutateConfig, config change, dimension mismatch land on next step) -- [x] Auto-update nesting: `update::run()` returns `Step`, included as substep +- [x] Pre-checks stay inline (MutateConfig, config change, dimension mismatch + land on next step) +- [x] Auto-update nesting: `update::run()` returns `Step`, included as + substep - [x] Violations: Validate substep carries violation data, Build aborts with Err - [x] Delete `BuildProcessOutput`, `BuildCommandOutput`, `BuildResult` structs ### main.rs + - [x] Update build match arm: `Step` dispatch ### Tests -- [x] Rewrite build tests with helpers (end-to-end, violations, incremental, model mismatch) + +- [x] Rewrite build tests with helpers (end-to-end, violations, incremental, + model mismatch) ### Verification + - [x] `cargo test` — all 313 tests pass - [x] `cargo clippy` + `cargo fmt` - [x] Manual: test against `example_kb/` with all output formats @@ -46,27 +60,36 @@ Convert `build` and `search` commands to the new Step tree architecture. Split i ## Wave 2: Search command ### Outcome structs (part of TODO-0122 phase 6) + - [x] Add `EmbedQuery`, `ExecuteSearch`, `Search` variants to both enums -- [x] Create `EmbedQueryOutcome` / `EmbedQueryOutcomeCompact` + `From` + `Render` -- [x] Create `ExecuteSearchOutcome` / `ExecuteSearchOutcomeCompact` + `From` + `Render` +- [x] Create `EmbedQueryOutcome` / `EmbedQueryOutcomeCompact` + `From` + + `Render` +- [x] Create `ExecuteSearchOutcome` / `ExecuteSearchOutcomeCompact` + `From` + + `Render` - [x] Create `SearchOutcome` / `SearchOutcomeCompact` + `From` + `Render` - [x] Create `SearchHitCompact` + `From<&SearchHit>` - Skipped: ReadChunkText — chunk text populated inline, not a separate step ### Search command rewrite + - [x] Rewrite `search::run()` → returns `Step` -- [x] Auto-build nesting: `build::run()` returns `Step`, included as substep +- [x] Auto-build nesting: `build::run()` returns `Step`, included as + substep - [x] Chunk text always populated, compact drops it -- [x] Delete `SearchProcessOutput`, `SearchCommandOutput`, `SearchResult` structs +- [x] Delete `SearchProcessOutput`, `SearchCommandOutput`, `SearchResult` + structs ### main.rs + - [x] Update search match arm: `Step` dispatch - [x] Removed unused `CommandOutput` import (all commands now use Step) ### Tests + - [x] Rewrite search tests with helpers ### Final verification + - [x] `cargo test` — all 313 tests pass - [x] `cargo clippy` — clean - [x] `cargo fmt` — formatted @@ -74,15 +97,20 @@ Convert `build` and `search` commands to the new Step tree architecture. Split i ## Files ### Wave 1 + - `src/cmd/build.rs` (rewrite run(), delete old structs, rewrite tests) - `src/outcome/commands/build.rs` (new) -- `src/outcome/` leaf files (new outcomes for MutateConfig, ReadIndexMetadata, CheckConfigChanged, Classify, LoadModel, EmbedFiles, WriteIndex) +- `src/outcome/` leaf files (new outcomes for MutateConfig, ReadIndexMetadata, + CheckConfigChanged, Classify, LoadModel, EmbedFiles, WriteIndex) - `src/outcome/commands/mod.rs` + `src/outcome/mod.rs` (add variants) - `src/main.rs` (update build match arm) ### Wave 2 + - `src/cmd/search.rs` (rewrite run(), delete old structs, rewrite tests) - `src/outcome/commands/search.rs` (new) -- `src/outcome/` leaf files (new outcomes for EmbedQuery, ExecuteSearch, ReadChunkText) -- `src/outcome/commands/mod.rs` + `src/outcome/mod.rs` (add variants, remove catch-alls) +- `src/outcome/` leaf files (new outcomes for EmbedQuery, ExecuteSearch, + ReadChunkText) +- `src/outcome/commands/mod.rs` + `src/outcome/mod.rs` (add variants, remove + catch-alls) - `src/main.rs` (update search match arm) diff --git a/docs/spec/todos/TODO-0131.md b/docs/spec/todos/TODO-0131.md index 749047c..5ef4efa 100644 --- a/docs/spec/todos/TODO-0131.md +++ b/docs/spec/todos/TODO-0131.md @@ -13,126 +13,176 @@ completed: 2026-03-23 ## Summary -Full cleanup: rewrite all 7 commands to call core functions directly (replacing `from_pipeline_result(run_*(...))` with `ScannedFiles::scan()` + timing + Step construction). Delete entire `src/pipeline/` directory. Remove all migration glue from `step.rs`. Clean up `output.rs`. +Full cleanup: rewrite all 7 commands to call core functions directly (replacing +`from_pipeline_result(run_*(...))` with `ScannedFiles::scan()` + timing + Step +construction). Delete entire `src/pipeline/` directory. Remove all migration +glue from `step.rs`. Clean up `output.rs`. ## Strategy -Each pipeline `run_*()` function wraps a core function with timing + `ProcessingStepResult`. Commands currently call `from_pipeline_result(run_*(), ...)` to convert. The cleanup eliminates both layers — commands call core functions directly and construct Steps inline. +Each pipeline `run_*()` function wraps a core function with timing + +`ProcessingStepResult`. Commands currently call +`from_pipeline_result(run_*(), ...)` to convert. The cleanup eliminates both +layers — commands call core functions directly and construct Steps inline. -A `Step::leaf()` constructor helper reduces the boilerplate of building leaf steps. +A `Step::leaf()` constructor helper reduces the boilerplate of building leaf +steps. ## Wave 1: Infrastructure + dead code -- [ ] Add `Step::leaf(outcome, elapsed_ms)` and `Step::failed(kind, message, elapsed_ms)` constructors to `step.rs` -- [ ] Move `BuildFileDetail` from `src/pipeline/write_index.rs` to `src/output.rs` -- [ ] Delete dead code from `src/output.rs`: `CommandOutput` trait, `format_json_compact()` -- [ ] Delete `StepOutput` trait from `src/pipeline/mod.rs` (if unused outside pipeline) +- [ ] Add `Step::leaf(outcome, elapsed_ms)` and + `Step::failed(kind, message, elapsed_ms)` constructors to `step.rs` +- [ ] Move `BuildFileDetail` from `src/pipeline/write_index.rs` to + `src/output.rs` +- [ ] Delete dead code from `src/output.rs`: `CommandOutput` trait, + `format_json_compact()` +- [ ] Delete `StepOutput` trait from `src/pipeline/mod.rs` (if unused outside + pipeline) ### Verification + - [ ] `cargo build` + `cargo test` + `cargo clippy` ## Wave 2: clean + info (simplest commands) ### clean -- [ ] Rewrite to call `Backend::clean()` / `walk_dir_stats()` directly (from `pipeline/delete_index.rs`) + +- [ ] Rewrite to call `Backend::clean()` / `walk_dir_stats()` directly (from + `pipeline/delete_index.rs`) - [ ] Delete `src/pipeline/delete_index.rs` ### info -- [ ] Rewrite to call `MdvsToml::read()` directly (from `pipeline/read_config.rs`) + +- [ ] Rewrite to call `MdvsToml::read()` directly (from + `pipeline/read_config.rs`) - [ ] Call `ScannedFiles::scan()` directly (from `pipeline/scan.rs`) - [ ] Call `backend.read_*()` directly (from `pipeline/read_index.rs`) -- [ ] Delete pipeline modules IF no other command still uses them (likely not yet — scan/read_config used by many) +- [ ] Delete pipeline modules IF no other command still uses them (likely not + yet — scan/read_config used by many) ### Verification + - [ ] `cargo build` + `cargo test` + `cargo clippy` ## Wave 3: init + update (scan/infer/write_config users) ### init + - [ ] Call `ScannedFiles::scan()` directly - [ ] Call `DirectoryTree::infer()` directly (from `pipeline/infer.rs`) - [ ] Call `MdvsToml::write()` directly (from `pipeline/write_config.rs`) ### update + - [ ] Same core functions as init + field comparison inline ### Delete pipeline modules + - [ ] Delete `src/pipeline/infer.rs` (no more callers) - [ ] Delete `src/pipeline/write_config.rs` (no more callers) ### Verification + - [ ] `cargo build` + `cargo test` + `cargo clippy` ## Wave 4: check (scan/validate user) ### check + - [ ] Call `ScannedFiles::scan()` directly -- [ ] Call `check::validate()` directly (already a core function, not a pipeline wrapper — just remove `run_validate` indirection) +- [ ] Call `check::validate()` directly (already a core function, not a pipeline + wrapper — just remove `run_validate` indirection) ### Delete pipeline modules + - [ ] Delete `src/pipeline/validate.rs` (no more callers) ### Verification + - [ ] `cargo build` + `cargo test` + `cargo clippy` ## Wave 5: build (most complex — scan/validate/classify/load_model/embed/write_index) ### build + - [ ] Call `ScannedFiles::scan()` directly - [ ] Call `check::validate()` directly -- [ ] Call `classify_files()` directly (from `pipeline/classify.rs` — move core logic) +- [ ] Call `classify_files()` directly (from `pipeline/classify.rs` — move core + logic) - [ ] Call `Embedder::load()` directly (from `pipeline/load_model.rs`) - [ ] Call `embed_files()` directly (from `pipeline/embed.rs` — move core logic) -- [ ] Call write_index logic directly (from `pipeline/write_index.rs` — move core logic) +- [ ] Call write_index logic directly (from `pipeline/write_index.rs` — move + core logic) ### Delete pipeline modules + - [ ] Delete `src/pipeline/classify.rs` - [ ] Delete `src/pipeline/load_model.rs` - [ ] Delete `src/pipeline/embed.rs` - [ ] Delete `src/pipeline/write_index.rs` ### Verification + - [ ] `cargo build` + `cargo test` + `cargo clippy` ## Wave 6: search (read_index/load_model/embed_query/execute_search) ### search + - [ ] Call `MdvsToml::read()` directly - [ ] Call `backend.read_*()` directly - [ ] Call `Embedder::load()` directly -- [ ] Call `embedder.embed()` directly (from `pipeline/embed.rs` run_embed_query) -- [ ] Call `SearchContext::execute()` directly (from `pipeline/execute_search.rs`) +- [ ] Call `embedder.embed()` directly (from `pipeline/embed.rs` + run_embed_query) +- [ ] Call `SearchContext::execute()` directly (from + `pipeline/execute_search.rs`) ### Delete pipeline modules + - [ ] Delete `src/pipeline/execute_search.rs` - [ ] Delete `src/pipeline/read_index.rs` - [ ] Delete `src/pipeline/read_config.rs` - [ ] Delete `src/pipeline/scan.rs` ### Verification + - [ ] `cargo build` + `cargo test` + `cargo clippy` ## Wave 7: Final deletion - [ ] Delete `src/pipeline/mod.rs` - [ ] Remove `pub mod pipeline` from `src/lib.rs` -- [ ] Delete migration helpers from `src/step.rs`: `from_pipeline_result`, `from_pipeline_result_with_data`, `convert_error_kind`, `has_failed_step()` compat method -- [ ] Remove `OutputFormat` from `output.rs` if moved elsewhere, or keep if still used +- [ ] Delete migration helpers from `src/step.rs`: `from_pipeline_result`, + `from_pipeline_result_with_data`, `convert_error_kind`, + `has_failed_step()` compat method +- [ ] Remove `OutputFormat` from `output.rs` if moved elsewhere, or keep if + still used - [ ] `cargo fmt` ### Final verification + - [ ] `cargo test` — all tests pass - [ ] `cargo clippy` — no warnings - [ ] `cargo fmt` — formatted -- [ ] Manual end-to-end: test all 7 commands against `example_kb/` with all output formats +- [ ] Manual end-to-end: test all 7 commands against `example_kb/` with all + output formats ## Notes -- Pipeline modules that contain meaningful core logic (classify's `classify_files()`, embed's batch logic, write_index's parquet assembly) need their logic moved to `src/index/` or kept as free functions in the command. Don't lose the logic — just unwrap it from the `ProcessingStepResult` wrapping. -- `read_config` and `scan` are the most widely shared — they'll be the last pipeline modules deleted (Wave 6). -- `check::validate()` is already a core function in `src/cmd/check.rs` — `pipeline/validate.rs` is just a thin wrapper. Easy delete. -- Some pipeline modules (`read_config.rs`, `scan.rs`) call core functions that are trivial (`MdvsToml::read()`, `ScannedFiles::scan()`). These are one-liners to inline. +- Pipeline modules that contain meaningful core logic (classify's + `classify_files()`, embed's batch logic, write_index's parquet assembly) need + their logic moved to `src/index/` or kept as free functions in the command. + Don't lose the logic — just unwrap it from the `ProcessingStepResult` + wrapping. +- `read_config` and `scan` are the most widely shared — they'll be the last + pipeline modules deleted (Wave 6). +- `check::validate()` is already a core function in `src/cmd/check.rs` — + `pipeline/validate.rs` is just a thin wrapper. Easy delete. +- Some pipeline modules (`read_config.rs`, `scan.rs`) call core functions that + are trivial (`MdvsToml::read()`, `ScannedFiles::scan()`). These are one-liners + to inline. ## Files affected -All `src/cmd/*.rs`, all `src/pipeline/*.rs` (deleted), `src/step.rs`, `src/output.rs`, `src/lib.rs` +All `src/cmd/*.rs`, all `src/pipeline/*.rs` (deleted), `src/step.rs`, +`src/output.rs`, `src/lib.rs` diff --git a/docs/spec/todos/TODO-0132.md b/docs/spec/todos/TODO-0132.md index ff8ef31..ebf0855 100644 --- a/docs/spec/todos/TODO-0132.md +++ b/docs/spec/todos/TODO-0132.md @@ -13,11 +13,17 @@ completed: 2026-03-23 ## Summary -Use `crabtime` to auto-generate `*Compact` structs and `From` impls from annotated full structs. Deferred until the manual pattern from TODO-0119 is proven stable. +Use `crabtime` to auto-generate `*Compact` structs and `From` impls from +annotated full structs. Deferred until the manual pattern from TODO-0119 is +proven stable. ## Details -After the Step tree architecture is fully implemented (TODO-0131 complete), the ~34 `From` impls and ~24 compact structs represent mechanical boilerplate. A `crabtime` macro could generate compact structs by stripping fields marked with a `#[compact_skip]` attribute and auto-generating the `From<&Full> for Compact` impl. +After the Step tree architecture is fully implemented (TODO-0131 complete), the +~34 `From` impls and ~24 compact structs represent mechanical boilerplate. A +`crabtime` macro could generate compact structs by stripping fields marked with +a `#[compact_skip]` attribute and auto-generating the `From<&Full> for Compact` +impl. ### Scope diff --git a/docs/spec/todos/TODO-0133.md b/docs/spec/todos/TODO-0133.md index 9bb7c54..0dd2ab2 100644 --- a/docs/spec/todos/TODO-0133.md +++ b/docs/spec/todos/TODO-0133.md @@ -14,8 +14,12 @@ subsumed_by: 139 ## Original Scope -Create a declarative macro to reduce the repetitive early-return pattern in command `run()` functions. +Create a declarative macro to reduce the repetitive early-return pattern in +command `run()` functions. ## Resolution -Subsumed by [TODO-0139](TODO-0139.md). Analysis showed a macro is not justified — the 39 pattern instances vary too much (different error kinds, data-dependent outcome construction, nested matches, different return types). A shared fail helper on `CommandResult` is the better approach. +Subsumed by [TODO-0139](TODO-0139.md). Analysis showed a macro is not justified +— the 39 pattern instances vary too much (different error kinds, data-dependent +outcome construction, nested matches, different return types). A shared fail +helper on `CommandResult` is the better approach. diff --git a/docs/spec/todos/TODO-0134.md b/docs/spec/todos/TODO-0134.md index 4655470..cf8c347 100644 --- a/docs/spec/todos/TODO-0134.md +++ b/docs/spec/todos/TODO-0134.md @@ -13,32 +13,37 @@ blocks: [] ## Summary -Audit findings from the Step tree migration (TODO-0131). Covers dead parameter cleanup, unused imports, and missing unit tests for moved functions. +Audit findings from the Step tree migration (TODO-0131). Covers dead parameter +cleanup, unused imports, and missing unit tests for moved functions. ## Details ### 1. `fail_from_last_skip()` unused parameter (build.rs) -The function accepts `_skipped: usize` but ignores it. Remove the parameter — callers always pass `0`. (The broader Skipped-padding question is tracked in TODO-0135.) +The function accepts `_skipped: usize` but ignores it. Remove the parameter — +callers always pass `0`. (The broader Skipped-padding question is tracked in +TODO-0135.) ### 2. Unused test imports (update.rs) -Test module imports `ViolationKind`, `FieldsConfig`, `UpdateConfig`, `FieldTypeSerde`, `ScanConfig` but doesn't use them. Lint cleanup. +Test module imports `ViolationKind`, `FieldsConfig`, `UpdateConfig`, +`FieldTypeSerde`, `ScanConfig` but doesn't use them. Lint cleanup. ### 3. Missing unit tests for private functions Functions moved during the pipeline cleanup that lack isolation tests: -| Function | File | Currently tested via | -|----------|------|---------------------| -| `validate_where_clause()` | `src/cmd/search.rs` | Integration only | -| `read_lines()` | `src/cmd/search.rs` | Not tested | -| `has_failed()` | `src/step.rs` | Indirect only | -| `has_violations()` | `src/step.rs` | Indirect only | -| `type_matches()` | `src/cmd/check.rs` | Integration only | -| `matches_any_glob()` | `src/cmd/check.rs` | Integration only | - -`embed_file()` in build.rs is excluded — requires a real Embedder and is well-covered by integration tests. +| Function | File | Currently tested via | +| ------------------------- | ------------------- | -------------------- | +| `validate_where_clause()` | `src/cmd/search.rs` | Integration only | +| `read_lines()` | `src/cmd/search.rs` | Not tested | +| `has_failed()` | `src/step.rs` | Indirect only | +| `has_violations()` | `src/step.rs` | Indirect only | +| `type_matches()` | `src/cmd/check.rs` | Integration only | +| `matches_any_glob()` | `src/cmd/check.rs` | Integration only | + +`embed_file()` in build.rs is excluded — requires a real Embedder and is +well-covered by integration tests. ### Deferred to TODO-0135 diff --git a/docs/spec/todos/TODO-0135.md b/docs/spec/todos/TODO-0135.md index f1dfc6f..8328aea 100644 --- a/docs/spec/todos/TODO-0135.md +++ b/docs/spec/todos/TODO-0135.md @@ -13,13 +13,18 @@ completed: 2026-03-23 ## Summary -Commands currently pad their substep list with `Step::skipped()` entries on early failure so the tree always has a fixed number of substeps. This is a leftover from the old fixed-position pipeline model and is unnecessary with the Step tree architecture, where each step is identified by its Outcome type, not its position. +Commands currently pad their substep list with `Step::skipped()` entries on +early failure so the tree always has a fixed number of substeps. This is a +leftover from the old fixed-position pipeline model and is unnecessary with the +Step tree architecture, where each step is identified by its Outcome type, not +its position. ## Details ### Current behavior -When a command fails early, it pushes anonymous Skipped entries for every step that won't run: +When a command fails early, it pushes anonymous Skipped entries for every step +that won't run: ```rust // build fails after scan → pad with 5 Skipped @@ -27,22 +32,33 @@ return fail_from_last(&mut substeps, start, 5); // result: [ReadConfig, Scan(failed), Skipped, Skipped, Skipped, Skipped, Skipped] ``` -Every error path in every command has a hardcoded skip count. These counts are fragile (the audit found inconsistencies when auto-update/auto-build nesting changes the total). +Every error path in every command has a hardcoded skip count. These counts are +fragile (the audit found inconsistencies when auto-update/auto-build nesting +changes the total). ### Proposed behavior -Stop padding. A failed-after-scan tree is just `[ReadConfig, Scan(failed)]`. The renderer shows what happened and stops. Each step is identified by its `Outcome` variant, not by position. +Stop padding. A failed-after-scan tree is just `[ReadConfig, Scan(failed)]`. The +renderer shows what happened and stops. Each step is identified by its `Outcome` +variant, not by position. ### Scope -1. **Remove skip-count parameters** from `fail_from_last`, `fail_msg`, `fail_early` helpers in all commands +1. **Remove skip-count parameters** from `fail_from_last`, `fail_msg`, + `fail_early` helpers in all commands 2. **Stop pushing Skipped substeps** on error paths -3. **Update rendering** to handle variable-length substep lists (currently rendering uses `Render` trait on each substep's Outcome — should already work, but verify) -4. **Attach auto-update/auto-build steps** as substeps in check.rs (currently dropped on success), matching build.rs and search.rs patterns -5. **Verify JSON output** still makes sense without padding (compact and verbose) +3. **Update rendering** to handle variable-length substep lists (currently + rendering uses `Render` trait on each substep's Outcome — should already + work, but verify) +4. **Attach auto-update/auto-build steps** as substeps in check.rs (currently + dropped on success), matching build.rs and search.rs patterns +5. **Verify JSON output** still makes sense without padding (compact and + verbose) ### Related - check.rs drops auto-update output on success (fix as part of this) -- build.rs/search.rs auto-update/auto-build nesting complicates skip counts (resolved by removing padding) -- `fail_from_last_skip()` in build.rs has unused `_skipped` parameter (resolved by this TODO) +- build.rs/search.rs auto-update/auto-build nesting complicates skip counts + (resolved by removing padding) +- `fail_from_last_skip()` in build.rs has unused `_skipped` parameter (resolved + by this TODO) diff --git a/docs/spec/todos/TODO-0136.md b/docs/spec/todos/TODO-0136.md index 1dad124..7d88abf 100644 --- a/docs/spec/todos/TODO-0136.md +++ b/docs/spec/todos/TODO-0136.md @@ -13,7 +13,10 @@ completed: 2026-03-23 ## Summary -Commands that call other commands' `run()` functions (check → update, build → update, search → build) cause redundant config reads and directory scans. Inline the subcommand logic so each command reads config once, scans once, and passes data forward. +Commands that call other commands' `run()` functions (check → update, build → +update, search → build) cause redundant config reads and directory scans. Inline +the subcommand logic so each command reads config once, scans once, and passes +data forward. ## Problem @@ -23,15 +26,20 @@ Current behavior with auto flags enabled: - `mdvs build`: 3 config reads, 2 full scans - `mdvs search` (auto_build + auto_update): 5+ config reads, 3+ scans -Each command calls another command's `run()` as a black box, which starts from scratch. +Each command calls another command's `run()` as a black box, which starts from +scratch. ## Solution -Commands inline the subcommand logic using shared data. Core functions (`ScannedFiles::scan()`, `InferredSchema::infer()`, `check::validate()`, etc.) are already available — commands just need to wire them together without redundant reads. +Commands inline the subcommand logic using shared data. Core functions +(`ScannedFiles::scan()`, `InferredSchema::infer()`, `check::validate()`, etc.) +are already available — commands just need to wire them together without +redundant reads. ## Wave 1: check (auto_update) **Current flow:** + 1. Read config 2. Call `update::run(path, ...)` → reads config, scans, infers, writes config 3. Re-read config @@ -39,9 +47,11 @@ Commands inline the subcommand logic using shared data. Core functions (`Scanned 5. Validate **New flow:** + 1. Read config 2. Scan (once) -3. If auto_update: infer → compare fields → write config if changed → re-read config +3. If auto_update: infer → compare fields → write config if changed → re-read + config 4. Validate (using same scanned files) **Result:** 1 config read (+ 1 re-read if config changed), 1 scan. @@ -51,6 +61,7 @@ Commands inline the subcommand logic using shared data. Core functions (`Scanned ## Wave 2: build (auto_update) **Current flow:** + 1. Read config 2. Call `update::run(path, ...)` → reads config, scans, infers, writes config 3. Re-read config @@ -58,9 +69,11 @@ Commands inline the subcommand logic using shared data. Core functions (`Scanned 5. Validate → classify → load model → embed → write index **New flow:** + 1. Read config 2. Scan (once) -3. If auto_update: infer → compare fields → write config if changed → re-read config +3. If auto_update: infer → compare fields → write config if changed → re-read + config 4. Validate (same scanned files) → classify → load model → embed → write index **Result:** 1 config read (+ 1 re-read if config changed), 1 scan. @@ -70,35 +83,41 @@ Commands inline the subcommand logic using shared data. Core functions (`Scanned ## Wave 3: search (auto_build) **Current flow:** + 1. Read config -2. Call `build::run(path, ...)` → reads config, calls update::run(), re-reads config, scans, validates, classifies, embeds, writes index +2. Call `build::run(path, ...)` → reads config, calls update::run(), re-reads + config, scans, validates, classifies, embeds, writes index 3. Re-read config 4. Read index → load model → embed query → execute search **New flow:** + 1. Read config -2. If auto_build: - a. Scan (once) - b. If auto_update: infer → compare fields → write config if changed → re-read config - c. Validate (same scanned files) → classify → load model → embed → write index -3. Read index → load model (reuse if already loaded) → embed query → execute search +2. If auto_build: a. Scan (once) b. If auto_update: infer → compare fields → + write config if changed → re-read config c. Validate (same scanned files) → + classify → load model → embed → write index +3. Read index → load model (reuse if already loaded) → embed query → execute + search -**Result:** 1 config read (+ 1 re-read if config changed), 1 scan, 1 model load (shared between build embed and search query embed). +**Result:** 1 config read (+ 1 re-read if config changed), 1 scan, 1 model load +(shared between build embed and search query embed). **Files:** `src/cmd/search.rs` ## Core functions available (no new code needed) -| Function | Module | Used by | -|----------|--------|---------| -| `MdvsToml::read()` + `.validate()` | `schema::config` | all | -| `ScannedFiles::scan()` | `discover::scan` | all | -| `InferredSchema::infer()` | `discover::infer` | check, build (auto_update) | -| `check::validate()` | `cmd::check` | check, build | -| `classify_files()` | `cmd::build` (private) | build, search (auto_build) | -| `embed_file()` | `cmd::build` (private) | build, search (auto_build) | -| `ModelConfig::try_from()` + `Embedder::load()` | `index::embed` | build, search | -| `Backend::write_index()` | `index::backend` | build, search (auto_build) | -| `Backend::search()` | `index::backend` | search | - -**Note:** `classify_files()` and `embed_file()` are currently private to build.rs. For wave 3 (search inlining build logic), these may need to move to a shared location or search must duplicate them. +| Function | Module | Used by | +| ---------------------------------------------- | ---------------------- | -------------------------- | +| `MdvsToml::read()` + `.validate()` | `schema::config` | all | +| `ScannedFiles::scan()` | `discover::scan` | all | +| `InferredSchema::infer()` | `discover::infer` | check, build (auto_update) | +| `check::validate()` | `cmd::check` | check, build | +| `classify_files()` | `cmd::build` (private) | build, search (auto_build) | +| `embed_file()` | `cmd::build` (private) | build, search (auto_build) | +| `ModelConfig::try_from()` + `Embedder::load()` | `index::embed` | build, search | +| `Backend::write_index()` | `index::backend` | build, search (auto_build) | +| `Backend::search()` | `index::backend` | search | + +**Note:** `classify_files()` and `embed_file()` are currently private to +build.rs. For wave 3 (search inlining build logic), these may need to move to a +shared location or search must duplicate them. diff --git a/docs/spec/todos/TODO-0137.md b/docs/spec/todos/TODO-0137.md index 08dbffd..13e7df2 100644 --- a/docs/spec/todos/TODO-0137.md +++ b/docs/spec/todos/TODO-0137.md @@ -13,11 +13,15 @@ completed: 2026-03-23 ## Summary -After TODO-0136 (inlining auto-update/auto-build), no command nests another command. The recursive `Step { substeps, outcome }` tree is always flat — substeps are always leaf nodes, never nested. Simplify the architecture to match reality: a flat list of process steps + a command result. +After TODO-0136 (inlining auto-update/auto-build), no command nests another +command. The recursive `Step { substeps, outcome }` tree is always flat — +substeps are always leaf nodes, never nested. Simplify the architecture to match +reality: a flat list of process steps + a command result. ## Problem Current JSON output has awkward nesting: + ```json { "substeps": [ @@ -29,40 +33,54 @@ Current JSON output has awkward nesting: ``` Issues: -- `substeps` always contains leaf nodes (empty `substeps: []`) — the recursive structure is unused -- `outcome` wraps another `outcome` — double nesting from `StepOutcome::Complete { result: Ok(Outcome::...) }` + +- `substeps` always contains leaf nodes (empty `substeps: []`) — the recursive + structure is unused +- `outcome` wraps another `outcome` — double nesting from + `StepOutcome::Complete { result: Ok(Outcome::...) }` - Verbose for consumers who just want the result ## Design decisions (settled) ### Compact vs verbose = "do you see the process steps or not?" -- **Compact** (default): command output only — summary line, tables, details. Everything the command produces. -- **Verbose** (`-v`): process step lines (ReadConfig, Scan, Validate, etc.) shown above the command output. +- **Compact** (default): command output only — summary line, tables, details. + Everything the command produces. +- **Verbose** (`-v`): process step lines (ReadConfig, Scan, Validate, etc.) + shown above the command output. -The `-v` flag controls visibility of process steps. It does NOT change what data the command outcome contains. +The `-v` flag controls visibility of process steps. It does NOT change what data +the command outcome contains. ### Text and JSON show the same information -Whatever is visible in text is also present in JSON, and vice versa. No hidden data in either format. +Whatever is visible in text is also present in JSON, and vice versa. No hidden +data in either format. ### Error mode is always verbose -When a command fails with a technical error (not validation), output is always verbose regardless of `-v`. The user needs to see which step failed. +When a command fails with a technical error (not validation), output is always +verbose regardless of `-v`. The user needs to see which step failed. ### CompactOutcome is deleted -No separate compact enum. Verbose just means "also serialize/render the steps." The command result struct is always the same — compact JSON is just the result struct serialized directly. +No separate compact enum. Verbose just means "also serialize/render the steps." +The command result struct is always the same — compact JSON is just the result +struct serialized directly. -This kills ~1,000 lines: all compact outcome structs, From impls, to_compact() conversion, and duplicate Render impls. +This kills ~1,000 lines: all compact outcome structs, From impls, to_compact() +conversion, and duplicate Render impls. ### Render trait stays -Every outcome type implements `Render` → `Vec`. In verbose mode, each step's outcome is rendered. In compact mode, only the command outcome is rendered. The Render trait is how each step self-manages its own output. +Every outcome type implements `Render` → `Vec`. In verbose mode, each +step's outcome is rendered. In compact mode, only the command outcome is +rendered. The Render trait is how each step self-manages its own output. ### Failed steps include elapsed_ms Useful for debugging (tells you if the step timed out or failed instantly): + ```json { "type": "Scan", "status": "failed", "error": "permission denied", "elapsed_ms": 0 } ``` @@ -72,6 +90,7 @@ Useful for debugging (tells you if the step timed out or failed instantly): ### Compact (default) Just the command result — no wrapper: + ```json { "files_checked": 43, "violations": [], "new_fields": [] } ``` @@ -79,6 +98,7 @@ Just the command result — no wrapper: ### Verbose (-v) Steps + result + timing: + ```json { "steps": [ @@ -106,18 +126,26 @@ Steps + result + timing: ## Scope -1. Delete `CompactOutcome` enum + all compact outcome structs + `to_compact()` + compact Render impls (~1,000 lines) -2. Replace `Step { substeps, outcome }` with flat `CommandResult { steps, result, elapsed_ms }` -3. Replace `StepOutcome` with simpler per-step serialization (type tag + fields) -4. Update custom Serialize: compact JSON = just result struct; verbose JSON = steps + result + elapsed +1. Delete `CompactOutcome` enum + all compact outcome structs + `to_compact()` + + compact Render impls (~1,000 lines) +2. Replace `Step { substeps, outcome }` with flat + `CommandResult { steps, result, elapsed_ms }` +3. Replace `StepOutcome` with simpler per-step serialization (type tag + + fields) +4. Update custom Serialize: compact JSON = just result struct; verbose JSON = + steps + result + elapsed 5. Update `has_failed()` / `has_violations()` for flat structure 6. Update all 7 commands to build `Vec` + command result -7. Update main.rs dispatch (compact vs verbose controls what to serialize/render) +7. Update main.rs dispatch (compact vs verbose controls what to + serialize/render) 8. Update Render impls (remove compact duplicates, keep full impls) 9. Update all tests ## Subsumed TODOs This TODO likely subsumes: -- **TODO-0132** (compact struct generation macro) — no longer needed if CompactOutcome is deleted -- **TODO-0135** (Skipped padding removal) — flat structure has no padding concept + +- **TODO-0132** (compact struct generation macro) — no longer needed if + CompactOutcome is deleted +- **TODO-0135** (Skipped padding removal) — flat structure has no padding + concept diff --git a/docs/spec/todos/TODO-0138.md b/docs/spec/todos/TODO-0138.md index a437dee..557aba0 100644 --- a/docs/spec/todos/TODO-0138.md +++ b/docs/spec/todos/TODO-0138.md @@ -13,11 +13,15 @@ blocks: [] ## Summary -The `Outcome` enum serializes with variant names as JSON keys: `{ "Check": { "files_checked": 43, ... } }`. The consumer already knows which command they ran — the wrapper is noise. Serialize the inner struct directly: `{ "files_checked": 43, ... }`. +The `Outcome` enum serializes with variant names as JSON keys: +`{ "Check": { "files_checked": 43, ... } }`. The consumer already knows which +command they ran — the wrapper is noise. Serialize the inner struct directly: +`{ "files_checked": 43, ... }`. ## Problem Current verbose JSON: + ```json { "steps": [ @@ -30,6 +34,7 @@ Current verbose JSON: ``` Desired: + ```json { "steps": [ @@ -41,24 +46,32 @@ Desired: } ``` -Same for compact JSON — currently `{ "Check": { ... } }`, should be `{ "files_checked": 43, ... }`. +Same for compact JSON — currently `{ "Check": { ... } }`, should be +`{ "files_checked": 43, ... }`. ## Approach -Add `#[serde(untagged)]` to the `Outcome` enum in `src/outcome/mod.rs`. This tells serde to serialize the inner struct directly without the variant name wrapper. +Add `#[serde(untagged)]` to the `Outcome` enum in `src/outcome/mod.rs`. This +tells serde to serialize the inner struct directly without the variant name +wrapper. ## Impact - Changes verbose JSON: step outcomes lose variant name keys - Changes compact JSON: result loses variant name key - Consumers parsing `json["result"]["Check"]` must change to `json["result"]` -- Step type is no longer identifiable from JSON alone (but each struct has distinct fields) +- Step type is no longer identifiable from JSON alone (but each struct has + distinct fields) ## Design question -Without the variant name, a consumer can't tell which step type produced an outcome just from the JSON. Is that acceptable? Each step's outcome struct has unique fields (e.g., `files_found` + `glob` = Scan, `config_path` = ReadConfig), but it's implicit rather than explicit. +Without the variant name, a consumer can't tell which step type produced an +outcome just from the JSON. Is that acceptable? Each step's outcome struct has +unique fields (e.g., `files_found` + `glob` = Scan, `config_path` = ReadConfig), +but it's implicit rather than explicit. Alternative: keep a `"type"` field alongside the flattened fields: + ```json { "status": "complete", "elapsed_ms": 5, "type": "ReadConfig", "config_path": "..." } ``` diff --git a/docs/spec/todos/TODO-0139.md b/docs/spec/todos/TODO-0139.md index 91243fa..cb547b0 100644 --- a/docs/spec/todos/TODO-0139.md +++ b/docs/spec/todos/TODO-0139.md @@ -13,21 +13,25 @@ blocks: [] ## Summary -Each command defines its own fail helper(s) for constructing a failed `CommandResult` from the steps list. These are copy-pasted with minor variations. Extract a shared helper into `step.rs`. +Each command defines its own fail helper(s) for constructing a failed +`CommandResult` from the steps list. These are copy-pasted with minor +variations. Extract a shared helper into `step.rs`. ## Problem Current state — 5 separate implementations across commands: -| File | Helpers | Error extraction | -|------|---------|-----------------| -| `init.rs` | `fail_early`, `fail_from_last_substep` | `.last()` | -| `update.rs` | `fail_early`, `fail_from_last_substep` | `.last()` (identical to init) | -| `build.rs` | `fail_from_last` | `.iter().rev().find_map()` | -| `search.rs` | `fail_from_last`, `fail_msg` | `.iter().rev().find_map()` | -| `check.rs`, `info.rs`, `clean.rs` | none (inline) | inline construction | +| File | Helpers | Error extraction | +| --------------------------------- | -------------------------------------- | ----------------------------- | +| `init.rs` | `fail_early`, `fail_from_last_substep` | `.last()` | +| `update.rs` | `fail_early`, `fail_from_last_substep` | `.last()` (identical to init) | +| `build.rs` | `fail_from_last` | `.iter().rev().find_map()` | +| `search.rs` | `fail_from_last`, `fail_msg` | `.iter().rev().find_map()` | +| `check.rs`, `info.rs`, `clean.rs` | none (inline) | inline construction | -The init/update helpers are byte-for-byte identical. The build/search helpers are nearly identical but use a better error extraction strategy (reverse search vs last). +The init/update helpers are byte-for-byte identical. The build/search helpers +are nearly identical but use a better error extraction strategy (reverse search +vs last). ## Solution @@ -61,15 +65,21 @@ impl CommandResult { ``` Then replace per-command helpers: -- `fail_from_last(&mut steps, start)` → `CommandResult::failed_from_steps(std::mem::take(&mut steps), start)` -- `fail_early(steps, start, kind, msg)` → `CommandResult::failed(steps, kind, msg, start)` -- `fail_msg(&mut steps, start, kind, msg)` → `CommandResult::failed(std::mem::take(&mut steps), kind, msg.into(), start)` + +- `fail_from_last(&mut steps, start)` → + `CommandResult::failed_from_steps(std::mem::take(&mut steps), start)` +- `fail_early(steps, start, kind, msg)` → + `CommandResult::failed(steps, kind, msg, start)` +- `fail_msg(&mut steps, start, kind, msg)` → + `CommandResult::failed(std::mem::take(&mut steps), kind, msg.into(), start)` - Inline constructions → use the appropriate method ## Files to modify -- `src/step.rs` — add `failed_from_steps` and `failed` constructors on `CommandResult` -- `src/cmd/init.rs` — delete `fail_early`, `fail_from_last_substep`, use shared methods +- `src/step.rs` — add `failed_from_steps` and `failed` constructors on + `CommandResult` +- `src/cmd/init.rs` — delete `fail_early`, `fail_from_last_substep`, use shared + methods - `src/cmd/update.rs` — same - `src/cmd/build.rs` — delete `fail_from_last`, use shared method - `src/cmd/search.rs` — delete `fail_from_last`, `fail_msg`, use shared methods diff --git a/docs/spec/todos/TODO-0140.md b/docs/spec/todos/TODO-0140.md index f49ad41..e8192f2 100644 --- a/docs/spec/todos/TODO-0140.md +++ b/docs/spec/todos/TODO-0140.md @@ -12,7 +12,9 @@ blocks: [] ## Summary -Make `--dry-run` a global CLI flag (`#[arg(global = true)]`) available to all commands. Commands that write (init, update, build, clean) respect it. Read-only commands (check, search, info) ignore it silently. +Make `--dry-run` a global CLI flag (`#[arg(global = true)]`) available to all +commands. Commands that write (init, update, build, clean) respect it. Read-only +commands (check, search, info) ignore it silently. ## Current state diff --git a/docs/spec/todos/TODO-0141.md b/docs/spec/todos/TODO-0141.md index 3fbe038..d41c7cd 100644 --- a/docs/spec/todos/TODO-0141.md +++ b/docs/spec/todos/TODO-0141.md @@ -12,7 +12,9 @@ blocks: [] ## Summary -Add a global `--quiet` / `-q` flag that suppresses all output when a command succeeds. On error or violations, output is shown normally. Useful for CI/scripting where only the exit code matters. +Add a global `--quiet` / `-q` flag that suppresses all output when a command +succeeds. On error or violations, output is shown normally. Useful for +CI/scripting where only the exit code matters. ## Behavior @@ -25,4 +27,5 @@ Add a global `--quiet` / `-q` flag that suppresses all output when a command suc 1. Add `#[arg(global = true, short = 'q', long)]` `quiet: bool` to `Cli` struct 2. In main.rs dispatch: if `quiet && !failed && !violations` → skip printing -3. Conflicts with `--verbose` — clap can enforce mutual exclusivity via `conflicts_with` +3. Conflicts with `--verbose` — clap can enforce mutual exclusivity via + `conflicts_with` diff --git a/docs/spec/todos/TODO-0142.md b/docs/spec/todos/TODO-0142.md index 0fdfa62..6c8df8c 100644 --- a/docs/spec/todos/TODO-0142.md +++ b/docs/spec/todos/TODO-0142.md @@ -13,13 +13,23 @@ blocks: [] ## Summary -Chunk `start_line`/`end_line` values reference the full file (including frontmatter). When `read_lines()` in search.rs reads chunk text using these line numbers, frontmatter YAML is included in the output. The line numbers should be offset to reference the body only (after the `---` closing delimiter). +Chunk `start_line`/`end_line` values reference the full file (including +frontmatter). When `read_lines()` in search.rs reads chunk text using these line +numbers, frontmatter YAML is included in the output. The line numbers should be +offset to reference the body only (after the `---` closing delimiter). ## Problem -The chunker (`text-splitter`) operates on the body text extracted by `gray_matter`. It produces line numbers relative to the body. But when these are stored in `chunks.parquet`, they're stored as-is — not offset by the frontmatter's line count. When a chunk starts at the beginning of the body, `start_line` is 1, which in the full file is the `---` line. +The chunker (`text-splitter`) operates on the body text extracted by +`gray_matter`. It produces line numbers relative to the body. But when these are +stored in `chunks.parquet`, they're stored as-is — not offset by the +frontmatter's line count. When a chunk starts at the beginning of the body, +`start_line` is 1, which in the full file is the `---` line. + +Example: `projects/archived/gamma/post-mortem.md` has +`start_line: 1, end_line: 11`. The chunk text read back includes the +frontmatter: -Example: `projects/archived/gamma/post-mortem.md` has `start_line: 1, end_line: 11`. The chunk text read back includes the frontmatter: ``` --- title: "Project Gamma — Post-Mortem" @@ -29,16 +39,24 @@ status: archived ## Fix options -1. **Offset during chunking**: When building chunks, add the frontmatter line count to `start_line`/`end_line` before storing. The stored values then reference the full file correctly. +1. **Offset during chunking**: When building chunks, add the frontmatter line + count to `start_line`/`end_line` before storing. The stored values then + reference the full file correctly. -2. **Offset during read**: In `read_lines()`, detect and skip the frontmatter before reading. But this is fragile — the frontmatter might have changed since build time. +2. **Offset during read**: In `read_lines()`, detect and skip the frontmatter + before reading. But this is fragile — the frontmatter might have changed + since build time. -3. **Store body-relative lines + frontmatter offset**: Store `start_line`/`end_line` as body-relative, plus a `frontmatter_lines` field. Convert when reading. +3. **Store body-relative lines + frontmatter offset**: Store + `start_line`/`end_line` as body-relative, plus a `frontmatter_lines` field. + Convert when reading. -Option 1 is simplest and most correct — the stored line numbers should reference the actual file. +Option 1 is simplest and most correct — the stored line numbers should reference +the actual file. ## Files to investigate -- `src/cmd/build.rs` — `embed_file()` function, where `start_line`/`end_line` are set +- `src/cmd/build.rs` — `embed_file()` function, where `start_line`/`end_line` + are set - `src/index/chunk.rs` — `Chunks::new()`, how line numbers are computed - `src/cmd/search.rs` — `read_lines()`, where chunk text is read back diff --git a/docs/spec/todos/TODO-0143.md b/docs/spec/todos/TODO-0143.md index bdc2a84..6fbb24a 100644 --- a/docs/spec/todos/TODO-0143.md +++ b/docs/spec/todos/TODO-0143.md @@ -12,9 +12,13 @@ blocks: [] ## Summary -Meta-TODO for discussing potential constraint kinds beyond the three already planned (TODO-0006 `categories`, TODO-0008 `min`/`max`, TODO-0010 `min_length`/`max_length`). All would live in `[fields.field.constraints]` and follow the mutual exclusion rule: at most one constraint kind per field. +Meta-TODO for discussing potential constraint kinds beyond the three already +planned (TODO-0006 `categories`, TODO-0008 `min`/`max`, TODO-0010 +`min_length`/`max_length`). All would live in `[fields.field.constraints]` and +follow the mutual exclusion rule: at most one constraint kind per field. -None of these are approved for implementation — this is a list of candidates to evaluate. +None of these are approved for implementation — this is a list of candidates to +evaluate. ## Candidates @@ -25,7 +29,9 @@ None of these are approved for implementation — this is a list of candidates t pattern = "^[a-z0-9-]+$" ``` -Validates that the string value matches a regex. Use cases: slugs, identifiers, semantic versioning, ISO codes. Applies to String only. Array(String) would check each element. +Validates that the string value matches a regex. Use cases: slugs, identifiers, +semantic versioning, ISO codes. Applies to String only. Array(String) would +check each element. ### `unique` — cross-file uniqueness @@ -34,7 +40,10 @@ Validates that the string value matches a regex. Use cases: slugs, identifiers, unique = true ``` -No two files may share the same value for this field. Use cases: `slug`, `id`, `permalink`. This is different from the other constraints — it's cross-file rather than single-value validation. Check would need to collect all values before reporting violations. +No two files may share the same value for this field. Use cases: `slug`, `id`, +`permalink`. This is different from the other constraints — it's cross-file +rather than single-value validation. Check would need to collect all values +before reporting violations. ### `unique_elements` — no duplicates within an array @@ -43,7 +52,8 @@ No two files may share the same value for this field. Use cases: `slug`, `id`, ` unique_elements = true ``` -Each element in the array must be distinct within that file's array. Use case: `tags` shouldn't have `["rust", "rust"]`. Applies to Array types only. +Each element in the array must be distinct within that file's array. Use case: +`tags` shouldn't have `["rust", "rust"]`. Applies to Array types only. ### `format` — semantic string format @@ -52,12 +62,23 @@ Each element in the array must be distinct within that file's array. Use case: ` format = "date" # or "url", "email", "path" ``` -Predefined format validators for common string patterns. Lighter than regex, covers the 80% case. `"date"` could be a stepping stone toward a proper Date type (TODO-0007). Applies to String only. +Predefined format validators for common string patterns. Lighter than regex, +covers the 80% case. `"date"` could be a stepping stone toward a proper Date +type (TODO-0007). Applies to String only. ## Open questions -- **`unique` is cross-file**: all other constraints validate a single value in isolation. `unique` requires collecting all values first. Does it belong in the same `[constraints]` section, or is it a different kind of field attribute? -- **`unique_elements` is array-structural**: it constrains the array shape, not the element values. Same question — is it a constraint or a field attribute? -- **Mutual exclusion scope**: should `unique` and `unique_elements` participate in mutual exclusion? They don't conflict with value constraints (`categories`, `min`/`max`) in the same way value constraints conflict with each other. A field could meaningfully be both `unique` across files AND have `categories`. -- **`format` vs `pattern`**: do we need both? `format` is a curated set of patterns. Could `format` just be sugar for common `pattern` values? -- **Which of these are actually useful?** Evaluate against real vaults before committing to any. +- **`unique` is cross-file**: all other constraints validate a single value in + isolation. `unique` requires collecting all values first. Does it belong in + the same `[constraints]` section, or is it a different kind of field + attribute? +- **`unique_elements` is array-structural**: it constrains the array shape, not + the element values. Same question — is it a constraint or a field attribute? +- **Mutual exclusion scope**: should `unique` and `unique_elements` participate + in mutual exclusion? They don't conflict with value constraints (`categories`, + `min`/`max`) in the same way value constraints conflict with each other. A + field could meaningfully be both `unique` across files AND have `categories`. +- **`format` vs `pattern`**: do we need both? `format` is a curated set of + patterns. Could `format` just be sugar for common `pattern` values? +- **Which of these are actually useful?** Evaluate against real vaults before + committing to any. diff --git a/docs/spec/todos/TODO-0144.md b/docs/spec/todos/TODO-0144.md index e80af62..e56d961 100644 --- a/docs/spec/todos/TODO-0144.md +++ b/docs/spec/todos/TODO-0144.md @@ -12,15 +12,22 @@ blocks: [] ## Summary -Embed a scripting runtime into the mdvs CLI to allow users to define custom transforms, validators, computed fields, and hooks. This turns mdvs from a fixed CLI into a programmable markdown database — the core handles parsing, Arrow, Parquet, embeddings, and SQL; scripting is the escape hatch for everything domain-specific. +Embed a scripting runtime into the mdvs CLI to allow users to define custom +transforms, validators, computed fields, and hooks. This turns mdvs from a fixed +CLI into a programmable markdown database — the core handles parsing, Arrow, +Parquet, embeddings, and SQL; scripting is the escape hatch for everything +domain-specific. -**Post-1.0.** Ship without scripting first, observe what users actually request, then decide if scripting earns its weight. If only one hook point gets requested (e.g. computed fields), solve it narrowly instead. +**Post-1.0.** Ship without scripting first, observe what users actually request, +then decide if scripting earns its weight. If only one hook point gets requested +(e.g. computed fields), solve it narrowly instead. ## Hook points Six well-defined extension points, mapped to existing pipeline stages: ### 1. Custom validators (cross-field, conditional) + ``` -- "published" requires "date" to be set function validate(fields) @@ -31,7 +38,8 @@ end ``` ### 2. Computed fields (derived columns in parquet, not in frontmatter) -``` + +```` function compute(file) return { word_count = count_words(file.body), @@ -39,19 +47,23 @@ function compute(file) has_code = file.body:match("```") ~= nil, } end -``` +```` + Queryable via `--where "word_count > 500 AND has_code = true"`. ### 3. Content transform before embedding -``` + +```` function pre_embed(body) -- strip all code blocks before embedding return body:gsub("```.-```", "") end -``` +```` + Controls what the search index sees without changing source files. ### 4. Custom reranking (post-search score adjustment) + ``` function rerank(results, query) for _, r in ipairs(results) do @@ -64,6 +76,7 @@ end ``` ### 5. Custom field extraction (beyond YAML frontmatter) + ``` function extract(file) if not file.fields.title then @@ -73,45 +86,76 @@ end ``` ### 6. Export / report generation + ``` function export(files) -- generate a JSON feed, RSS, tag cloud, link graph... end ``` + mdvs already has all data loaded — scripts consume it in arbitrary ways. ## Language evaluation (March 2026) ### Lua (via `mlua`) -- **Pros:** 30 years of precedent (Redis, Neovim, Nginx, WezTerm). Massive ecosystem, universal familiarity, tiny VM (~300KB), excellent perf (LuaJIT ~2-5x C). -- **Cons:** Compiles from C source — adds `build.rs` and C compiler dependency, contradicts supply chain hardening stance. 1-indexed arrays, fragmented ecosystem (5.1/5.2/5.3/5.4/LuaJIT all subtly incompatible). -- **Verdict:** Strongest general choice, but the C dependency is a real cost for us. + +- **Pros:** 30 years of precedent (Redis, Neovim, Nginx, WezTerm). Massive + ecosystem, universal familiarity, tiny VM (~300KB), excellent perf (LuaJIT + ~2-5x C). +- **Cons:** Compiles from C source — adds `build.rs` and C compiler dependency, + contradicts supply chain hardening stance. 1-indexed arrays, fragmented + ecosystem (5.1/5.2/5.3/5.4/LuaJIT all subtly incompatible). +- **Verdict:** Strongest general choice, but the C dependency is a real cost for + us. ### Rhai -- **Pros:** Pure Rust (no C, no build scripts). Designed specifically for Rust embedding. Rust-like syntax. Sandboxed by default. ~5.6M crate downloads, multiple contributors, well-documented. -- **Cons:** No JIT, ~10-50x C perf. Less expressive than Lua (by design — it's a config/scripting language, not a general-purpose one). -- **Verdict:** Boring-correct choice. Strongest adoption in the pure-Rust embedding space. + +- **Pros:** Pure Rust (no C, no build scripts). Designed specifically for Rust + embedding. Rust-like syntax. Sandboxed by default. ~5.6M crate downloads, + multiple contributors, well-documented. +- **Cons:** No JIT, ~10-50x C perf. Less expressive than Lua (by design — it's a + config/scripting language, not a general-purpose one). +- **Verdict:** Boring-correct choice. Strongest adoption in the pure-Rust + embedding space. ### Starlark (via `starlark-rust`) -- **Pros:** Pure Rust. Python-like syntax (zero learning curve). Deterministic execution by design (no infinite loops, no recursion, no I/O). Backed by Meta (used in Buck2). -- **Cons:** Restrictions that make it safe also make it limiting — no long-running computations, no direct I/O. API is more complex (designed for build systems, not general embedding). -- **Verdict:** Best sandboxing story. Good if we want to guarantee script termination. + +- **Pros:** Pure Rust. Python-like syntax (zero learning curve). Deterministic + execution by design (no infinite loops, no recursion, no I/O). Backed by Meta + (used in Buck2). +- **Cons:** Restrictions that make it safe also make it limiting — no + long-running computations, no direct I/O. API is more complex (designed for + build systems, not general embedding). +- **Verdict:** Best sandboxing story. Good if we want to guarantee script + termination. ### Rune -- **Pros:** Pure Rust. Rust-like syntax (`fn`, `let`, `match`, `async`/`await`). Native async support — scripts can await Rust futures. Proper module system. -- **Cons:** Solo maintainer (bus factor 1, ~97% of commits from one person). Pre-1.0, still in flux. ~141K downloads (40x fewer than Rhai). One verified production user (ScyllaDB's Latte benchmarker). Release cadence slowing (last release Sept 2025). -- **Verdict:** Technically fascinating, but too risky for a dependency. Revisit if it reaches 1.0 with broader adoption. + +- **Pros:** Pure Rust. Rust-like syntax (`fn`, `let`, `match`, `async`/`await`). + Native async support — scripts can await Rust futures. Proper module system. +- **Cons:** Solo maintainer (bus factor 1, ~97% of commits from one person). + Pre-1.0, still in flux. ~141K downloads (40x fewer than Rhai). One verified + production user (ScyllaDB's Latte benchmarker). Release cadence slowing (last + release Sept 2025). +- **Verdict:** Technically fascinating, but too risky for a dependency. Revisit + if it reaches 1.0 with broader adoption. ### Recommendation -**Rhai** if we want a straightforward scripting layer. **Starlark** if sandboxing and determinism matter more than expressiveness. Both are pure Rust with no C build dependencies. +**Rhai** if we want a straightforward scripting layer. **Starlark** if +sandboxing and determinism matter more than expressiveness. Both are pure Rust +with no C build dependencies. ### 7. Field transforms (before/after validation) Pydantic-style two-phase transforms on field values: -- **Before-validation transform** — normalize/clean the raw value before type checking and constraint validation runs. E.g., `" Draft "` → `"draft"` → passes `categories = ["draft", "published"]`. -- **After-validation transform** — transform the validated value before storage. E.g., slugify a title, normalize a date string. You know the value is valid, now shape it for querying. +- **Before-validation transform** — normalize/clean the raw value before type + checking and constraint validation runs. E.g., `" Draft "` → `"draft"` → + passes `categories = ["draft", "published"]`. +- **After-validation transform** — transform the validated value before storage. + E.g., slugify a title, normalize a date string. You know the value is valid, + now shape it for querying. ``` function before_validate(field_name, value) @@ -130,22 +174,34 @@ end ``` Pipeline per field value: + ``` raw YAML value → before_validate → type check + constraints → after_validate → stored value ``` Behavior by command: -- **check**: before-transforms run ephemerally (normalize → validate → discard), after-transforms skipped (nothing to store) -- **build**: both phases run — before normalizes for validation, after shapes for Parquet storage. `--where` queries see the after-transformed values. -This is distinct from content transforms (hook point 3), which operate on the markdown body for embedding. Field transforms operate on frontmatter values for validation and storage. +- **check**: before-transforms run ephemerally (normalize → validate → discard), + after-transforms skipped (nothing to store) +- **build**: both phases run — before normalizes for validation, after shapes + for Parquet storage. `--where` queries see the after-transformed values. + +This is distinct from content transforms (hook point 3), which operate on the +markdown body for embedding. Field transforms operate on frontmatter values for +validation and storage. ## Design questions to resolve -- **Where do scripts live?** Options: `.mdvs/scripts/`, a single `mdvs.rhai`, inline in `mdvs.toml`, or `[hooks]` section pointing to files. -- **Sandboxing:** restrict filesystem/network access? Both Rhai and Starlark are sandboxable. -- **Error model:** how do script errors surface? Warnings vs hard failures? Per-hook configuration? +- **Where do scripts live?** Options: `.mdvs/scripts/`, a single `mdvs.rhai`, + inline in `mdvs.toml`, or `[hooks]` section pointing to files. +- **Sandboxing:** restrict filesystem/network access? Both Rhai and Starlark are + sandboxable. +- **Error model:** how do script errors surface? Warnings vs hard failures? + Per-hook configuration? - **Performance:** computed fields run per-file during build. Benchmark needed. -- **API surface:** what exactly does the script see? Field values, file content, config, other files? Each exposure is a contract. -- **Relationship to TODO-0009:** does this subsume built-in processors, or do both coexist? -- **Relationship to TODO-0106 (link graph):** `extract` hook could parse wikilinks/internal links, feeding a graph. +- **API surface:** what exactly does the script see? Field values, file content, + config, other files? Each exposure is a contract. +- **Relationship to TODO-0009:** does this subsume built-in processors, or do + both coexist? +- **Relationship to TODO-0106 (link graph):** `extract` hook could parse + wikilinks/internal links, feeding a graph. diff --git a/docs/spec/todos/TODO-0145.md b/docs/spec/todos/TODO-0145.md index 2bb274d..d6465f8 100644 --- a/docs/spec/todos/TODO-0145.md +++ b/docs/spec/todos/TODO-0145.md @@ -14,8 +14,12 @@ subsumed_by: 149 ## Original Scope -Add a `pattern` constraint to String and Array(String) fields that validates values against a regex. +Add a `pattern` constraint to String and Array(String) fields that validates +values against a regex. ## Resolution -Subsumed by [TODO-0149](TODO-0149.md). Pattern constraints will be implemented as the JSON Schema `pattern` keyword with validation delegated to `jsonschema` (which uses `fancy-regex` for ECMA-262 regex). No hand-rolled validation code or direct `regex` crate dependency needed. +Subsumed by [TODO-0149](TODO-0149.md). Pattern constraints will be implemented +as the JSON Schema `pattern` keyword with validation delegated to `jsonschema` +(which uses `fancy-regex` for ECMA-262 regex). No hand-rolled validation code or +direct `regex` crate dependency needed. diff --git a/docs/spec/todos/TODO-0146.md b/docs/spec/todos/TODO-0146.md index f75880d..c16c5d0 100644 --- a/docs/spec/todos/TODO-0146.md +++ b/docs/spec/todos/TODO-0146.md @@ -13,32 +13,37 @@ blocks: [] ## Summary -The mdBook documentation needs two kinds of updates: fix stale `--reinfer`/`--reinfer-all` flag references (replaced by `update reinfer` subcommand in TODO-0006), and add new documentation for categorical constraints, the `InvalidCategory` violation, and the `[fields.field.constraints]` TOML section. +The mdBook documentation needs two kinds of updates: fix stale +`--reinfer`/`--reinfer-all` flag references (replaced by `update reinfer` +subcommand in TODO-0006), and add new documentation for categorical constraints, +the `InvalidCategory` violation, and the `[fields.field.constraints]` TOML +section. ## Details ### Wave 1: Fix stale `--reinfer` references -Replace all `--reinfer`/`--reinfer-all` flag syntax with the `update reinfer` subcommand syntax across 7 pages. Mechanical search-and-replace, no new content. +Replace all `--reinfer`/`--reinfer-all` flag syntax with the `update reinfer` +subcommand syntax across 7 pages. Mechanical search-and-replace, no new content. -| Page | What to fix | -|------|-------------| -| `book/src/commands/update.md` | Primary rewrite — replace flag table and sections with subcommand docs (`update reinfer [fields..] --categorical --no-categorical --max-categories --min-repetition --dry-run`) | -| `book/src/commands/init.md` | Fix `init --force` vs `update --reinfer-all` comparison → `update reinfer` | -| `book/src/configuration.md` | Fix `update --reinfer` reference | -| `book/src/concepts/schema.md` | Fix 4 references to `--reinfer` with examples | -| `book/src/concepts/types.md` | Fix 1 reference to `update --reinfer ` | -| `book/src/recipes/obsidian.md` | Fix 1 reference to `--reinfer` | +| Page | What to fix | +| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `book/src/commands/update.md` | Primary rewrite — replace flag table and sections with subcommand docs (`update reinfer [fields..] --categorical --no-categorical --max-categories --min-repetition --dry-run`) | +| `book/src/commands/init.md` | Fix `init --force` vs `update --reinfer-all` comparison → `update reinfer` | +| `book/src/configuration.md` | Fix `update --reinfer` reference | +| `book/src/concepts/schema.md` | Fix 4 references to `--reinfer` with examples | +| `book/src/concepts/types.md` | Fix 1 reference to `update --reinfer ` | +| `book/src/recipes/obsidian.md` | Fix 1 reference to `--reinfer` | ### Wave 2: New constraint documentation Create new page and update existing pages for categorical constraints. -| Action | Page | What | -|--------|------|------| +| Action | Page | What | +| ------ | ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | CREATE | `book/src/concepts/constraints.md` | New concepts page: what constraints are, the two-layer architecture (serde + behavior), categorical constraints (what they are, auto-inference heuristic with `max_categories`/`min_category_repetition`, manual override via `--categorical`/`--no-categorical`, TOML representation), future constraint kinds (range, length, pattern — brief mention) | -| UPDATE | `book/src/SUMMARY.md` | Add `constraints.md` entry under Concepts | -| UPDATE | `book/src/concepts.md` | Add link to new constraints page | -| UPDATE | `book/src/concepts/validation.md` | Add `InvalidCategory` as 5th violation type (rename "four violations" to "five violations") | -| UPDATE | `book/src/commands/check.md` | Add `InvalidCategory` violation to the violation types list with example | -| UPDATE | `book/src/configuration.md` | Document `[fields.field.constraints]` sub-table with `categories` key, document `max_categories` and `min_category_repetition` in `[fields]` section | +| UPDATE | `book/src/SUMMARY.md` | Add `constraints.md` entry under Concepts | +| UPDATE | `book/src/concepts.md` | Add link to new constraints page | +| UPDATE | `book/src/concepts/validation.md` | Add `InvalidCategory` as 5th violation type (rename "four violations" to "five violations") | +| UPDATE | `book/src/commands/check.md` | Add `InvalidCategory` violation to the violation types list with example | +| UPDATE | `book/src/configuration.md` | Document `[fields.field.constraints]` sub-table with `categories` key, document `max_categories` and `min_category_repetition` in `[fields]` section | diff --git a/docs/spec/todos/TODO-0147.md b/docs/spec/todos/TODO-0147.md index c8e9c70..1437544 100644 --- a/docs/spec/todos/TODO-0147.md +++ b/docs/spec/todos/TODO-0147.md @@ -13,25 +13,31 @@ blocks: [] ## Summary -The specs in `docs/spec/` have drifted from the code. More fundamentally, they try to be both a user manual and a developer map — the user manual parts (flag tables, output examples, TOML syntax) go stale instantly because they duplicate `--help` and the mdBook. The developer map parts (algorithms, data flow, design decisions) are the valuable part that doesn't exist elsewhere. +The specs in `docs/spec/` have drifted from the code. More fundamentally, they +try to be both a user manual and a developer map — the user manual parts (flag +tables, output examples, TOML syntax) go stale instantly because they duplicate +`--help` and the mdBook. The developer map parts (algorithms, data flow, design +decisions) are the valuable part that doesn't exist elsewhere. -Restructure the specs to serve one purpose: **a developer's map of how the code works**. Strip user-facing content (already in mdBook), focus on internal architecture, data flow, module responsibilities, and cross-cutting concerns. +Restructure the specs to serve one purpose: **a developer's map of how the code +works**. Strip user-facing content (already in mdBook), focus on internal +architecture, data flow, module responsibilities, and cross-cutting concerns. ## Current state -| Document | Lines | Status | Problem | -|----------|-------|--------|---------| -| `commands/init.md` | 147 | Very stale | Describes flags that don't exist | -| `commands/update.md` | 146 | Stale | Missing reinfer subcommand, FieldChange details | -| `shared.md` | 229 | Stale | Missing FieldChange, FieldHint; OutputFormat outdated | -| `commands/build.md` | 251 | Slightly stale | Missing `--no-update` flag | -| `commands/check.md` | 138 | Slightly stale | Missing `--no-update`, no InvalidCategory | -| `commands/search.md` | 231 | Slightly stale | Missing `--no-update`/`--no-build` | -| `commands/info.md` | 116 | Slightly stale | Missing `--no-update` | -| `commands/clean.md` | 31 | Status wrong | Marked DEFERRED but implemented | -| `output-format.md` | 511 | Current | Mostly user-facing (belongs in mdBook) | -| `cocogitto.md` | 103 | Current | Fine as-is (dev tooling reference) | -| `release.md` | 216 | Current | Fine as-is (dev process reference) | +| Document | Lines | Status | Problem | +| -------------------- | ----- | -------------- | ----------------------------------------------------- | +| `commands/init.md` | 147 | Very stale | Describes flags that don't exist | +| `commands/update.md` | 146 | Stale | Missing reinfer subcommand, FieldChange details | +| `shared.md` | 229 | Stale | Missing FieldChange, FieldHint; OutputFormat outdated | +| `commands/build.md` | 251 | Slightly stale | Missing `--no-update` flag | +| `commands/check.md` | 138 | Slightly stale | Missing `--no-update`, no InvalidCategory | +| `commands/search.md` | 231 | Slightly stale | Missing `--no-update`/`--no-build` | +| `commands/info.md` | 116 | Slightly stale | Missing `--no-update` | +| `commands/clean.md` | 31 | Status wrong | Marked DEFERRED but implemented | +| `output-format.md` | 511 | Current | Mostly user-facing (belongs in mdBook) | +| `cocogitto.md` | 103 | Current | Fine as-is (dev tooling reference) | +| `release.md` | 216 | Current | Fine as-is (dev process reference) | ## New structure @@ -63,13 +69,16 @@ docs/spec/ ### What each spec should contain **Architecture docs** (new top-level files): + - Module responsibilities: which files do what - Key types with source locations (`file:line`) - Data flow: input → processing stages → output - Design decisions with rationale -- Cross-cutting concerns (e.g., how constraints flow through init → config → check → build) +- Cross-cutting concerns (e.g., how constraints flow through init → config → + check → build) **Command specs** (rewritten): + - One paragraph: what the command does internally (not how to use it) - Data flow diagram: which modules are called in what order - Key decision points (e.g., build's incremental classification logic) @@ -79,20 +88,31 @@ docs/spec/ ### What moves to mdBook or gets deleted - CLI flag tables → already in mdBook command pages -- Output format examples → already in mdBook + `output-format.md` → move to mdBook +- Output format examples → already in mdBook + `output-format.md` → move to + mdBook - TOML syntax reference → already in mdBook configuration page ## Implementation waves ### Wave 1: `architecture.md` — the master map -The foundation. Every other doc references this. Create `docs/spec/architecture.md` covering: +The foundation. Every other doc references this. Create +`docs/spec/architecture.md` covering: - **Module tree**: every directory and file under `src/`, one-line purpose each -- **Key types**: the core structs/enums that flow through the pipeline (`ScannedFile`, `FieldType`, `InferredField`, `InferredSchema`, `MdvsToml`, `TomlField`, `Constraints`, `ConstraintKind`, `FileRow`, `ChunkRow`, `BuildMetadata`, `SearchContext`, etc.) with source locations (`file:line`) -- **Data pipeline**: end-to-end flow from `.md` files → frontmatter extraction → type inference → path inference → constraint inference → TOML config → validation → chunking → embedding → parquet storage → cosine search → DataFusion SQL -- **Key design decisions**: enum dispatch (not traits), two-layer constraint architecture, no lock file, incremental build via content_hash, internal column prefixing -- **Cross-cutting concerns**: how constraints flow through init → config → check → build, how auto-update chains commands +- **Key types**: the core structs/enums that flow through the pipeline + (`ScannedFile`, `FieldType`, `InferredField`, `InferredSchema`, `MdvsToml`, + `TomlField`, `Constraints`, `ConstraintKind`, `FileRow`, `ChunkRow`, + `BuildMetadata`, `SearchContext`, etc.) with source locations (`file:line`) +- **Data pipeline**: end-to-end flow from `.md` files → frontmatter extraction → + type inference → path inference → constraint inference → TOML config → + validation → chunking → embedding → parquet storage → cosine search → + DataFusion SQL +- **Key design decisions**: enum dispatch (not traits), two-layer constraint + architecture, no lock file, incremental build via content_hash, internal + column prefixing +- **Cross-cutting concerns**: how constraints flow through init → config → check + → build, how auto-update chains commands This single document should let a developer orient themselves in the codebase. @@ -100,15 +120,31 @@ This single document should let a developer orient themselves in the codebase. Focused architecture docs for each subsystem, referencing the master map: -- **`constraints.md`** — two-layer architecture (serde struct + enum dispatch), resolver (validate_config with self-validation + pairwise), per-kind submodule pattern, inference heuristic, how it wires into check -- **`inference.md`** — type inference (widening matrix), path inference (DirectoryTree → GlobMap → collapse), categorical inference, how FieldTypeInfo flows to InferredField to TomlField -- **`validation.md`** — check pipeline dispatch (check_field_values loop), violation accumulation (ViolationKey grouping), 5 violation types and their trigger conditions, constraint validation wiring -- **`storage.md`** — parquet layout (files.parquet + chunks.parquet), column constants, Arrow schema construction (data Struct column), BuildMetadata in native parquet key-value metadata, Backend enum, incremental build (FileClassification, content_hash) -- **`search.md`** — cosine similarity in Rust (not DataFusion UDF), SearchContext, files_v view creation for bare field names, internal_prefix + aliases, note-level ranking (max chunk similarity) +- **`constraints.md`** — two-layer architecture (serde struct + enum dispatch), + resolver (validate_config with self-validation + pairwise), per-kind submodule + pattern, inference heuristic, how it wires into check +- **`inference.md`** — type inference (widening matrix), path inference + (DirectoryTree → GlobMap → collapse), categorical inference, how FieldTypeInfo + flows to InferredField to TomlField +- **`validation.md`** — check pipeline dispatch (check_field_values loop), + violation accumulation (ViolationKey grouping), 5 violation types and their + trigger conditions, constraint validation wiring +- **`storage.md`** — parquet layout (files.parquet + chunks.parquet), column + constants, Arrow schema construction (data Struct column), BuildMetadata in + native parquet key-value metadata, Backend enum, incremental build + (FileClassification, content_hash) +- **`search.md`** — cosine similarity in Rust (not DataFusion UDF), + SearchContext, files_v view creation for bare field names, internal_prefix + + aliases, note-level ranking (max chunk similarity) ### Wave 3: Command spec rewrite + cleanup -- **Rewrite command specs** (`commands/*.md`): strip flag tables and output examples (already in mdBook), replace with internal data flow (which modules called in what order), key decision points, error handling strategy. One page per command, short and focused. -- **Update `shared.md`**: current types only with source locations, add missing types (FieldChange, FieldHint, ConstraintViolation) -- **Retire `output-format.md`**: content is user-facing and already covered by mdBook. Delete or move. +- **Rewrite command specs** (`commands/*.md`): strip flag tables and output + examples (already in mdBook), replace with internal data flow (which modules + called in what order), key decision points, error handling strategy. One page + per command, short and focused. +- **Update `shared.md`**: current types only with source locations, add missing + types (FieldChange, FieldHint, ConstraintViolation) +- **Retire `output-format.md`**: content is user-facing and already covered by + mdBook. Delete or move. - **Update `clean.md` status**: DEFERRED → DRAFT (command is implemented) diff --git a/docs/spec/todos/TODO-0148.md b/docs/spec/todos/TODO-0148.md index 69c8af1..8f848e4 100644 --- a/docs/spec/todos/TODO-0148.md +++ b/docs/spec/todos/TODO-0148.md @@ -12,24 +12,37 @@ blocks: [102] ## Summary -Add a CI step to the `book.yml` workflow that generates `llms.txt` and `llms-full.txt` from the mdBook content and deploys them alongside the book to GitHub Pages at `https://edochi.github.io/mdvs/llms.txt`. +Add a CI step to the `book.yml` workflow that generates `llms.txt` and +`llms-full.txt` from the mdBook content and deploys them alongside the book to +GitHub Pages at `https://edochi.github.io/mdvs/llms.txt`. ## Details -The [llms.txt convention](https://llmstxt.org/) provides LLM-friendly summaries of website content. The SKILL.md (TODO-0102) should reference this URL for deeper documentation context. +The [llms.txt convention](https://llmstxt.org/) provides LLM-friendly summaries +of website content. The SKILL.md (TODO-0102) should reference this URL for +deeper documentation context. ### What to generate -- **`llms.txt`** — book title + description + table of contents with links (one link per chapter/section). Derived from `book/src/SUMMARY.md`. -- **`llms-full.txt`** — book title + description + all chapter markdown concatenated. Full content for LLMs that want the complete docs. +- **`llms.txt`** — book title + description + table of contents with links (one + link per chapter/section). Derived from `book/src/SUMMARY.md`. +- **`llms-full.txt`** — book title + description + all chapter markdown + concatenated. Full content for LLMs that want the complete docs. ### How -A shell script step in `.github/workflows/book.yml`, after `mdbook build`, before the upload artifact step. No external dependencies — just parse `SUMMARY.md` and concatenate markdown files. The logic is trivial (see [mdbook-llms-txt-tools](https://github.com/higumachan/mdbook-llms-txt-tools) for reference — MIT licensed — but we don't need the crate, just a few lines of shell). +A shell script step in `.github/workflows/book.yml`, after `mdbook build`, +before the upload artifact step. No external dependencies — just parse +`SUMMARY.md` and concatenate markdown files. The logic is trivial (see +[mdbook-llms-txt-tools](https://github.com/higumachan/mdbook-llms-txt-tools) for +reference — MIT licensed — but we don't need the crate, just a few lines of +shell). -Output files go into `book/book/` (the mdBook output directory) so they're deployed with the rest of the site. +Output files go into `book/book/` (the mdBook output directory) so they're +deployed with the rest of the site. ### Files to update - `.github/workflows/book.yml` — add generation step -- `book/book.toml` — ensure `title` and `description` are set (needed for the header) +- `book/book.toml` — ensure `title` and `description` are set (needed for the + header) diff --git a/docs/spec/todos/TODO-0149.md b/docs/spec/todos/TODO-0149.md index cc87e77..dcfb0c8 100644 --- a/docs/spec/todos/TODO-0149.md +++ b/docs/spec/todos/TODO-0149.md @@ -26,36 +26,70 @@ related: > > **Wave A** (`tomljson` crate, workspace setup) — shipped. > -> **Wave B** (jsonschema engine, preprocessor pipeline, new CLI surface) — shipped, with one exception: **step 13** (per-file overlay synthesis for path-scoped validation) is **deferred** and bundled with [TODO-0154](./TODO-0154.md) (signature-keyed overlay cache); the naïve per-file compile cost makes the unoptimized form unacceptable at scale, so the two ship together. Path-scoping today runs Rust-side via `globset` — correct, just not the eventual architecture. Wave-B follow-up that landed in closeout: **strict-Float subtype precheck** (`preprocess::strict_subtype_check`) — `Float` / `Array(Float)` fields reject integer-backed values unless `widen_int_to_float` is in `preprocess`. Mirrors the `String` + `coerce_to_string` strictness pattern. Implemented in Rust rather than the JSON Schema layer because `jsonschema` can't see serde's i64/f64 distinction. +> **Wave B** (jsonschema engine, preprocessor pipeline, new CLI surface) — +> shipped, with one exception: **step 13** (per-file overlay synthesis for +> path-scoped validation) is **deferred** and bundled with +> [TODO-0154](./TODO-0154.md) (signature-keyed overlay cache); the naïve +> per-file compile cost makes the unoptimized form unacceptable at scale, so the +> two ship together. Path-scoping today runs Rust-side via `globset` — correct, +> just not the eventual architecture. Wave-B follow-up that landed in closeout: +> **strict-Float subtype precheck** (`preprocess::strict_subtype_check`) — +> `Float` / `Array(Float)` fields reject integer-backed values unless +> `widen_int_to_float` is in `preprocess`. Mirrors the `String` + +> `coerce_to_string` strictness pattern. Implemented in Rust rather than the +> JSON Schema layer because `jsonschema` can't see serde's i64/f64 distinction. > -> **Wave C** (object flattening, function-style type display, translator-Object-arm closure) — shipped under [TODO-0097](./TODO-0097.md). Type renames were dropped from the original plan (decided against during the design discussion — the cost of touching every reference outweighed the consistency gain). Wave C also folded in and closed [TODO-0096](./TODO-0096.md) (`String[]` → `Array(String)` display refresh). +> **Wave C** (object flattening, function-style type display, +> translator-Object-arm closure) — shipped under [TODO-0097](./TODO-0097.md). +> Type renames were dropped from the original plan (decided against during the +> design discussion — the cost of touching every reference outweighed the +> consistency gain). Wave C also folded in and closed +> [TODO-0096](./TODO-0096.md) (`String[]` → `Array(String)` display refresh). > > **Open follow-ups derived from this work:** -> - [TODO-0154](./TODO-0154.md) — overlay cache; bundle with Wave B step 13 when implemented. -> - [TODO-0155](./TODO-0155.md) — reusable type definitions (`$defs` / `$ref`); voluntary, medium priority. +> +> - [TODO-0154](./TODO-0154.md) — overlay cache; bundle with Wave B step 13 when +> implemented. +> - [TODO-0155](./TODO-0155.md) — reusable type definitions (`$defs` / `$ref`); +> voluntary, medium priority. ## Summary -Adopt JSON Schema 2020-12 as the canonical internal representation for field validation. Split into three internal waves: - -- **Wave A — `tomljson` crate**: new workspace member providing **lossless bidirectional translation between TOML and JSON-shaped data**, with explicit handling of TOML's impedance mismatches (no native null, no top-level non-table, signed-64-bit integer cap). Designed with JSON Schema 2020-12 as the motivating use case — anyone with arbitrary JSON data can use the same translator. No validation, no opinions, no JSON Schema awareness in the implementation. -- **Wave B — mdvs adoption**: mdvs uses JSON Schema internally for validation via the `jsonschema` crate. `mdvs.toml` becomes a user-friendly DSL that translates to canonical JSON Schema. Preprocessor pipeline handles leniency. Subsumes TODO-0008, TODO-0010, TODO-0145. -- **Wave C — type naming + object flattening alignment**: align `mdvs.toml` type names to JSON Schema (`string`, `integer`, `number`, `boolean`, `array`, `object`) and flatten nested objects per TODO-0097. +Adopt JSON Schema 2020-12 as the canonical internal representation for field +validation. Split into three internal waves: + +- **Wave A — `tomljson` crate**: new workspace member providing **lossless + bidirectional translation between TOML and JSON-shaped data**, with explicit + handling of TOML's impedance mismatches (no native null, no top-level + non-table, signed-64-bit integer cap). Designed with JSON Schema 2020-12 as + the motivating use case — anyone with arbitrary JSON data can use the same + translator. No validation, no opinions, no JSON Schema awareness in the + implementation. +- **Wave B — mdvs adoption**: mdvs uses JSON Schema internally for validation + via the `jsonschema` crate. `mdvs.toml` becomes a user-friendly DSL that + translates to canonical JSON Schema. Preprocessor pipeline handles leniency. + Subsumes TODO-0008, TODO-0010, TODO-0145. +- **Wave C — type naming + object flattening alignment**: align `mdvs.toml` type + names to JSON Schema (`string`, `integer`, `number`, `boolean`, `array`, + `object`) and flatten nested objects per TODO-0097. ## Design decisions (settled) ### Two-layer representation -| Layer | Form | Role | -|---|---|---| -| `tomljson` (canonical) | **Nested** TOML mirroring JSON Schema 1:1 — `[fields.field.properties.baseline.properties.wavelength]` | Pure translator. Full JSON Schema expressiveness. Reusable crate. | -| `mdvs.toml` (DSL) | **Flattened** `[[fields.field]]` with dotted names — `name = "calibration.baseline.wavelength"` (per TODO-0097) | User-friendly config surface. Translates to canonical JSON Schema internally. | +| Layer | Form | Role | +| ---------------------- | --------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| `tomljson` (canonical) | **Nested** TOML mirroring JSON Schema 1:1 — `[fields.field.properties.baseline.properties.wavelength]` | Pure translator. Full JSON Schema expressiveness. Reusable crate. | +| `mdvs.toml` (DSL) | **Flattened** `[[fields.field]]` with dotted names — `name = "calibration.baseline.wavelength"` (per TODO-0097) | User-friendly config surface. Translates to canonical JSON Schema internally. | -The flattened DSL is mdvs-specific UX. The canonical form is what `jsonschema` validates against. +The flattened DSL is mdvs-specific UX. The canonical form is what `jsonschema` +validates against. ### Type naming -Aligned to JSON Schema: `string`, `integer`, `number`, `boolean`, `array`, `object`, `null`. Replaces the current `String`, `Integer`, `Float`, `Boolean`, `Array(...)`, `Object({...})`. Wave C is the breaking config change. +Aligned to JSON Schema: `string`, `integer`, `number`, `boolean`, `array`, +`object`, `null`. Replaces the current `String`, `Integer`, `Float`, `Boolean`, +`Array(...)`, `Object({...})`. Wave C is the breaking config change. ### Validation pipeline @@ -88,33 +122,39 @@ Raw JSON value Everything per-value: -| Concern | JSON Schema keyword(s) | -|---|---| -| Type checking | `type` | -| Nullable | `"type": ["string", "null"]` (type union) | -| Categories | `enum` | -| Numeric bounds | `minimum`, `maximum` | -| String length | `minLength`, `maxLength` | -| Array length | `minItems`, `maxItems` | -| Regex pattern | `pattern` (ECMA-262 via `fancy-regex`) | -| Array element schema | `items` | -| Object structure | `properties` (recursive) | +| Concern | JSON Schema keyword(s) | +| -------------------- | ----------------------------------------- | +| Type checking | `type` | +| Nullable | `"type": ["string", "null"]` (type union) | +| Categories | `enum` | +| Numeric bounds | `minimum`, `maximum` | +| String length | `minLength`, `maxLength` | +| Array length | `minItems`, `maxItems` | +| Regex pattern | `pattern` (ECMA-262 via `fancy-regex`) | +| Array element schema | `items` | +| Object structure | `properties` (recursive) | ### What mdvs keeps (not delegated) -| Concern | Why | -|---|---| -| `validate_for_type` | Reject misapplied constraints at config load. JSON Schema silently ignores inapplicable keywords; mdvs wants a hard error. | -| `conflicts_with` | Reject contradictory constraints (e.g., `min > max`). | -| Path-aware rules | `allowed`/`required` globs are per-file-path, not per-value. Not expressible in JSON Schema. Stored under the property's `x-mdvs` extension; enforced at runtime by synthesizing a per-file overlay schema around `jsonschema::Validator` (see "Path-scoped validation" below). | -| Preprocessor pipeline | Value coercion/normalization before validation. mdvs domain logic. | -| Error mapping | Translate `jsonschema` errors to mdvs `ViolationKind` with human-readable messages. | +| Concern | Why | +| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `validate_for_type` | Reject misapplied constraints at config load. JSON Schema silently ignores inapplicable keywords; mdvs wants a hard error. | +| `conflicts_with` | Reject contradictory constraints (e.g., `min > max`). | +| Path-aware rules | `allowed`/`required` globs are per-file-path, not per-value. Not expressible in JSON Schema. Stored under the property's `x-mdvs` extension; enforced at runtime by synthesizing a per-file overlay schema around `jsonschema::Validator` (see "Path-scoped validation" below). | +| Preprocessor pipeline | Value coercion/normalization before validation. mdvs domain logic. | +| Error mapping | Translate `jsonschema` errors to mdvs `ViolationKind` with human-readable messages. | ### Preprocessor pipeline (three-stage) -The pipeline runs **between YAML→JSON conversion and `jsonschema` validation**. It has three discrete stages, each backed by an `enum` of preprocessor variants (matching the existing `Backend` / `Embedder` enum-dispatch pattern; closed sets with one Lua escape-hatch variant). All three stages skip in `--strict` mode (TODO-0150). +The pipeline runs **between YAML→JSON conversion and `jsonschema` validation**. +It has three discrete stages, each backed by an `enum` of preprocessor variants +(matching the existing `Backend` / `Embedder` enum-dispatch pattern; closed sets +with one Lua escape-hatch variant). All three stages skip in `--strict` mode +(TODO-0150). -**Configuration locality**: stages 1 and 3 are *global* (operate on keys / whole document, can't be per-field). Stage 2 is *per-field* and lives in each `[[fields.field]]` entry. See `mdvs.toml` shape below. +**Configuration locality**: stages 1 and 3 are _global_ (operate on keys / whole +document, can't be per-field). Stage 2 is _per-field_ and lives in each +`[[fields.field]]` entry. See `mdvs.toml` shape below. ``` Raw JSON value (post-YAML→JSON) @@ -146,7 +186,9 @@ Raw JSON value (post-YAML→JSON) jsonschema::Validator ``` -Stages are independent: each has its own trait, its own list of registered preprocessors, its own ordering. A user can register handlers in any combination. +Stages are independent: each has its own trait, its own list of registered +preprocessors, its own ordering. A user can register handlers in any +combination. #### Stage 1: Field-name preprocessors (global) @@ -174,7 +216,9 @@ impl FieldNamePreprocessor { } ``` -Practical use case: frontmatter has `Title: "Hello"` but the schema declares `title`. A `Lowercase` preprocessor closes the gap. Globally-applied; can't be per-field by definition. +Practical use case: frontmatter has `Title: "Hello"` but the schema declares +`title`. A `Lowercase` preprocessor closes the gap. Globally-applied; can't be +per-field by definition. #### Stage 2: Per-field value preprocessors @@ -206,19 +250,28 @@ impl FieldValuePreprocessor { } ``` -The preprocessor sees the field's individual JSON Schema sub-tree, extracted by the pipeline runner using the field name as a JSON pointer. +The preprocessor sees the field's individual JSON Schema sub-tree, extracted by +the pipeline runner using the field name as a JSON pointer. **Lenient-mode defaults** (always-on, type-driven, not user-configured): -- `CoerceToString` — applies to any field whose schema is `type: "string"` when the raw value isn't a string. -- `WidenIntToFloat` — applies to any field whose schema is `type: "number"` when the raw value is an integer. -These are baked into the default lenient behavior and don't appear in `mdvs.toml`. Strict mode skips them. +- `CoerceToString` — applies to any field whose schema is `type: "string"` when + the raw value isn't a string. +- `WidenIntToFloat` — applies to any field whose schema is `type: "number"` when + the raw value is an integer. -**Per-field opt-in** (declared on each `[[fields.field]]`): users name additional preprocessors to apply, in order. Built-ins are referred to by their kebab-case enum-variant names (`lowercase`, `trim-whitespace`, etc.); Lua preprocessors are referred to by their declared script name. +These are baked into the default lenient behavior and don't appear in +`mdvs.toml`. Strict mode skips them. + +**Per-field opt-in** (declared on each `[[fields.field]]`): users name +additional preprocessors to apply, in order. Built-ins are referred to by their +kebab-case enum-variant names (`lowercase`, `trim-whitespace`, etc.); Lua +preprocessors are referred to by their declared script name. #### Stage 3: Per-document preprocessors (global, future) -Operate on the whole frontmatter object, with full-schema visibility. Closed enum: +Operate on the whole frontmatter object, with full-schema visibility. Closed +enum: ```rust pub enum DocumentPreprocessor { @@ -229,11 +282,14 @@ pub enum DocumentPreprocessor { Use cases this stage exists to enable: -- **Cross-field coherence**: "if `status: published` then `published_at` must be set" — a preprocessor could fill in `published_at = today()` when missing. +- **Cross-field coherence**: "if `status: published` then `published_at` must be + set" — a preprocessor could fill in `published_at = today()` when missing. - **Computed defaults**: "if `slug` is missing, derive from `title`." -- **Field interdependencies**: "if `archived: true`, normalize `status: archived`." +- **Field interdependencies**: "if `archived: true`, normalize + `status: archived`." -Stage 3 has no built-ins in v0 of Wave B, but the enum and runner are defined so future variants don't require a pipeline rearchitecture. +Stage 3 has no built-ins in v0 of Wave B, but the enum and runner are defined so +future variants don't require a pipeline rearchitecture. #### Mode handling @@ -244,22 +300,33 @@ pub enum Mode { } ``` -`--strict` mode (TODO-0150): pipeline is bypassed entirely. The user's frontmatter must already match the schema exactly. +`--strict` mode (TODO-0150): pipeline is bypassed entirely. The user's +frontmatter must already match the schema exactly. #### Lua custom preprocessors (TODO-0144) -User-defined Lua scripts plug into any of the three stages via the `Lua(LuaScript)` enum variant. The script's name (declared once in `mdvs.toml`'s `[[preprocess.lua]]` array) is used in the per-field `preprocess` list; the runtime resolves the name to a `LuaScript` handle at config-load time. +User-defined Lua scripts plug into any of the three stages via the +`Lua(LuaScript)` enum variant. The script's name (declared once in `mdvs.toml`'s +`[[preprocess.lua]]` array) is used in the per-field `preprocess` list; the +runtime resolves the name to a `LuaScript` handle at config-load time. #### Why enum dispatch over `dyn Trait` -We considered `Box` for open extensibility. Rejected because: +We considered `Box` for open extensibility. Rejected +because: -- mdvs's user extension story is **Lua scripts**, not "downstream Rust crates implementing a trait." A single `Lua(LuaScript)` variant covers the entire user-extension surface. -- The set of built-in preprocessors is closed at the crate level; we add new ones in releases, not users. -- Match-based dispatch is a jump table; vtable indirection would cost ~ns per call across thousands of preprocessor invocations per `check`. -- The pattern matches the existing `Backend` and `Embedder` enums in mdvs — codebase consistency. +- mdvs's user extension story is **Lua scripts**, not "downstream Rust crates + implementing a trait." A single `Lua(LuaScript)` variant covers the entire + user-extension surface. +- The set of built-in preprocessors is closed at the crate level; we add new + ones in releases, not users. +- Match-based dispatch is a jump table; vtable indirection would cost ~ns per + call across thousands of preprocessor invocations per `check`. +- The pattern matches the existing `Backend` and `Embedder` enums in mdvs — + codebase consistency. -Trade-off: a downstream Rust crate cannot add its own preprocessor without forking mdvs. The Lua escape hatch is the answer. +Trade-off: a downstream Rust crate cannot add its own preprocessor without +forking mdvs. The Lua escape hatch is the answer. #### Pipeline runner @@ -306,7 +373,9 @@ impl Pipeline { } ``` -The per-field Stage 2 preprocessors are stored on the `CompiledSchema` (extracted from each property's `x-mdvs.preprocess` array at compile time), so the runner just looks them up by field name. +The per-field Stage 2 preprocessors are stored on the `CompiledSchema` +(extracted from each property's `x-mdvs.preprocess` array at compile time), so +the runner just looks them up by field name. #### `mdvs.toml` shape @@ -333,7 +402,8 @@ type = "array" preprocess = ["trim_whitespace", "normalize_my_tags"] ``` -Lenient defaults (`coerce_to_string`, `widen_int_to_float`) are not declared — they're always-on in lenient mode based on the field's type. +Lenient defaults (`coerce_to_string`, `widen_int_to_float`) are not declared — +they're always-on in lenient mode based on the field's type. ### Crate organization (Wave B reference layout) @@ -440,15 +510,20 @@ With `--schema ` (one-shot import): 5. Per file: preprocess pipeline → partition + per-file overlay → validate ``` -In override mode the schema is the sole source of field/preprocess rules; mdvs.toml's `[fields]`/`[preprocess]` sections are ignored even when present (other sections — `[scan]` etc. — still apply). +In override mode the schema is the sole source of field/preprocess rules; +mdvs.toml's `[fields]`/`[preprocess]` sections are ignored even when present +(other sections — `[scan]` etc. — still apply). #### `update [path]` -Unchanged shape; inference logic gains length/pattern support (constraint inference module). **No `--schema` flag** — `update` re-infers from the filesystem only. +Unchanged shape; inference logic gains length/pattern support (constraint +inference module). **No `--schema` flag** — `update` re-infers from the +filesystem only. #### `build [path]` -Calls `check` internally (existing). The check call now goes through the new pipeline. **No `--schema` flag** — build always reads mdvs.toml. +Calls `check` internally (existing). The check call now goes through the new +pipeline. **No `--schema` flag** — build always reads mdvs.toml. #### `search`, `info`, `clean` @@ -466,19 +541,19 @@ Untouched. ### Module migration map -| Concern | Today | After Wave B | -|---|---|---| -| YAML→JSON conversion | `discover/scan.rs` (silent .ok()?) | `discover/scan.rs` (explicit error) | -| Type validation | `schema/constraints/*::validate_value` | delegated to `jsonschema::Validator` | -| Constraint-on-value validation | hand-rolled per kind | delegated to `jsonschema::Validator` | -| `validate_for_type` (config-load checks) | `schema/constraints/*::validate_for_type` | **stays** | -| `conflicts_with` (constraint compatibility) | `schema/constraints/mod.rs` | **stays** | -| Path-aware rules (allowed/required) | `cmd/check.rs` | **stays** | -| DSL ↔ JSON Schema translation | does not exist | `schema/json_schema.rs` | -| Compiled validator | does not exist | `schema/validator.rs` | -| Value preprocessing | does not exist | `preprocess/` (3 stages) | -| Constraint inference | `discover/infer/constraints/` | **stays**, expanded with length/pattern | -| TOML I/O of schema documents | does not exist | `tomljson::{from_str, to_string}` | +| Concern | Today | After Wave B | +| ------------------------------------------- | ----------------------------------------- | --------------------------------------- | +| YAML→JSON conversion | `discover/scan.rs` (silent .ok()?) | `discover/scan.rs` (explicit error) | +| Type validation | `schema/constraints/*::validate_value` | delegated to `jsonschema::Validator` | +| Constraint-on-value validation | hand-rolled per kind | delegated to `jsonschema::Validator` | +| `validate_for_type` (config-load checks) | `schema/constraints/*::validate_for_type` | **stays** | +| `conflicts_with` (constraint compatibility) | `schema/constraints/mod.rs` | **stays** | +| Path-aware rules (allowed/required) | `cmd/check.rs` | **stays** | +| DSL ↔ JSON Schema translation | does not exist | `schema/json_schema.rs` | +| Compiled validator | does not exist | `schema/validator.rs` | +| Value preprocessing | does not exist | `preprocess/` (3 stages) | +| Constraint inference | `discover/infer/constraints/` | **stays**, expanded with length/pattern | +| TOML I/O of schema documents | does not exist | `tomljson::{from_str, to_string}` | ### Wave C deltas to the layout @@ -487,20 +562,31 @@ Wave C (TODO-0097 + lowercase type names) touches: - `discover/field_type.rs` — variant rename (`String` → `string`, etc.) - `schema/shared.rs` — re-exports - `schema/config.rs` — flattened `[[fields.field]]` form with dotted names -- `schema/json_schema.rs` — translator handles dotted names → nested `properties` +- `schema/json_schema.rs` — translator handles dotted names → nested + `properties` - `book/`, `docs/spec/`, `example_kb/mdvs.toml` — content updates -Mechanical once Waves A and B are stable. Breaking config change → its own release. +Mechanical once Waves A and B are stable. Breaking config change → its own +release. ### YAML → JSON conversion (frontmatter → validatable value) -mdvs reads YAML frontmatter from `.md` files and validates the resulting *JSON-shaped* value against a JSON Schema. JSON Schema validators (including the `jsonschema` crate) operate on `serde_json::Value`, not on YAML. The YAML→JSON conversion is therefore an unavoidable step before validation can happen. +mdvs reads YAML frontmatter from `.md` files and validates the resulting +_JSON-shaped_ value against a JSON Schema. JSON Schema validators (including the +`jsonschema` crate) operate on `serde_json::Value`, not on YAML. The YAML→JSON +conversion is therefore an unavoidable step before validation can happen. -**Asymmetry with the TOML side**: TOML and JSON have a structural impedance mismatch (TOML lacks null, top-level non-table values, and integers > i64::MAX) that requires a dedicated bridging crate (`tomljson`). YAML and JSON are much closer — JSON 1.0 is a strict subset of YAML 1.2 — so `serde_yaml`'s built-in `Pod::deserialize::()` handles ~95% of the conversion. There is no need for a `yamljson` companion crate. +**Asymmetry with the TOML side**: TOML and JSON have a structural impedance +mismatch (TOML lacks null, top-level non-table values, and integers > i64::MAX) +that requires a dedicated bridging crate (`tomljson`). YAML and JSON are much +closer — JSON 1.0 is a strict subset of YAML 1.2 — so `serde_yaml`'s built-in +`Pod::deserialize::()` handles ~95% of the conversion. There +is no need for a `yamljson` companion crate. **Where the YAML→JSON gap leaks in mdvs today** -The current code at `src/discover/scan.rs:126` swallows conversion failures silently: +The current code at `src/discover/scan.rs:126` swallows conversion failures +silently: ```rust let data = parsed.data.and_then(|d: Pod| { @@ -509,47 +595,67 @@ let data = parsed.data.and_then(|d: Pod| { }); ``` -If any single field has a value that can't fit in `serde_json::Value` (the only realistic cases: `.inf` / `-.inf` / `.nan` floats, or non-string mapping keys), `Pod::deserialize` returns an error, `.ok()?` discards it, and the **entire frontmatter** is treated as missing. Other valid fields in the same file are also lost. +If any single field has a value that can't fit in `serde_json::Value` (the only +realistic cases: `.inf` / `-.inf` / `.nan` floats, or non-string mapping keys), +`Pod::deserialize` returns an error, `.ok()?` discards it, and the **entire +frontmatter** is treated as missing. Other valid fields in the same file are +also lost. **Required Wave B work** Replace the silent drop with explicit handling: -| YAML feature | Wave B handling | -|---|---| -| `.inf` / `-.inf` / `.nan` float | Error with `ViolationKind::FrontmatterUnrepresentable { field, reason }` indicating which field and why. Same policy as `tomljson`'s decode side. | -| Non-string mapping keys | Error with same kind; very rare in real frontmatter. | -| Anchors / aliases | Resolved by `serde_yaml` at parse time → already JSON-compatible. No action needed. | -| Tags (`!!str`, custom) | Flattened by `serde_yaml` to base types. No action needed. | +| YAML feature | Wave B handling | +| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `.inf` / `-.inf` / `.nan` float | Error with `ViolationKind::FrontmatterUnrepresentable { field, reason }` indicating which field and why. Same policy as `tomljson`'s decode side. | +| Non-string mapping keys | Error with same kind; very rare in real frontmatter. | +| Anchors / aliases | Resolved by `serde_yaml` at parse time → already JSON-compatible. No action needed. | +| Tags (`!!str`, custom) | Flattened by `serde_yaml` to base types. No action needed. | | Implicit dates (`published: 2026-05-04`) | Kept as strings in YAML 1.2 mode (`serde_yaml` default). No action needed unless we want to coerce to a normalized form (preprocessor candidate). | -| Comments | Discarded at parse time. Not data, no action. | -| Multi-document `---` | Not applicable to single-document frontmatter. | +| Comments | Discarded at parse time. Not data, no action. | +| Multi-document `---` | Not applicable to single-document frontmatter. | The fix is two parts: -1. **In `scan.rs`**: surface the deserialize error instead of `.ok()?`. Identify which field caused the failure if possible (may require a custom `Deserialize` walker since `Pod::deserialize` to a generic Value returns an opaque error). -2. **In the validation layer (Wave B)**: define a new `ViolationKind` variant for unrepresentable frontmatter values, distinct from "frontmatter missing" or "value of wrong type." +1. **In `scan.rs`**: surface the deserialize error instead of `.ok()?`. Identify + which field caused the failure if possible (may require a custom + `Deserialize` walker since `Pod::deserialize` to a generic Value returns an + opaque error). +2. **In the validation layer (Wave B)**: define a new `ViolationKind` variant + for unrepresentable frontmatter values, distinct from "frontmatter missing" + or "value of wrong type." -This is a small but real correctness improvement that the JSON Schema integration enables — once we adopt `jsonschema`, we're committing to "the value must be representable as JSON," and silent drops contradict that promise. +This is a small but real correctness improvement that the JSON Schema +integration enables — once we adopt `jsonschema`, we're committing to "the value +must be representable as JSON," and silent drops contradict that promise. -**Documentation requirement (book/)**: the user guide for the validation feature must explain that frontmatter values are validated against JSON-shaped data, and must list the YAML→JSON conversion rules so users understand why `.inf` isn't accepted. +**Documentation requirement (book/)**: the user guide for the validation feature +must explain that frontmatter values are validated against JSON-shaped data, and +must list the YAML→JSON conversion rules so users understand why `.inf` isn't +accepted. ### Schema sourcing (`--schema` flag) -`--schema PATH` accepts a JSON Schema file. It is exposed on **two commands only**, with two distinct semantics: +`--schema PATH` accepts a JSON Schema file. It is exposed on **two commands +only**, with two distinct semantics: -| Command | Semantics with `--schema` | Without `--schema` | -|---|---|---| -| `init [PATH] --schema FILE` | **One-shot import.** Translate FILE → `mdvs.toml`'s inline DSL. FILE is consumed, not stored. | Scan + infer (current behavior). | -| `check [PATH] --schema FILE` | **Sole source of truth.** Validate using FILE for fields/preprocess. `mdvs.toml` may be absent; defaults fill in scan config. | Use `mdvs.toml` (required). | +| Command | Semantics with `--schema` | Without `--schema` | +| ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | -------------------------------- | +| `init [PATH] --schema FILE` | **One-shot import.** Translate FILE → `mdvs.toml`'s inline DSL. FILE is consumed, not stored. | Scan + infer (current behavior). | +| `check [PATH] --schema FILE` | **Sole source of truth.** Validate using FILE for fields/preprocess. `mdvs.toml` may be absent; defaults fill in scan config. | Use `mdvs.toml` (required). | -`--schema` is **not** accepted on `update`, `build`, `search`, `info`, or `clean`. Reasoning per command in the section below. +`--schema` is **not** accepted on `update`, `build`, `search`, `info`, or +`clean`. Reasoning per command in the section below. #### `mdvs.toml` shape (no `schema_ref`) -There is no `schema_ref` field in `mdvs.toml`. The toml is the single source of truth for the project (scan, embedding model, chunking, search, fields, preprocess). External schemas are either imported once at `init` time (becoming inline DSL) or supplied per invocation to `check` (not persisted). +There is no `schema_ref` field in `mdvs.toml`. The toml is the single source of +truth for the project (scan, embedding model, chunking, search, fields, +preprocess). External schemas are either imported once at `init` time (becoming +inline DSL) or supplied per invocation to `check` (not persisted). -The earlier draft had a 3-tier resolution priority (CLI override → `schema_ref` → inline DSL). That is dropped. Resolution is now binary: +The earlier draft had a 3-tier resolution priority (CLI override → `schema_ref` +→ inline DSL). That is dropped. Resolution is now binary: ```rust fn resolve_schema(cli_override: Option<&Path>, mdvs_toml: Option<&MdvsToml>) -> Result { @@ -565,88 +671,130 @@ fn resolve_schema(cli_override: Option<&Path>, mdvs_toml: Option<&MdvsToml>) -> The schema-loading helper sniffs by file extension: -| Extension | Loader | -|---|---| -| `.json` | `serde_json::from_str` | -| `.toml` | `tomljson::from_str` (canonical schema in TOML form) | -| `.yaml`, `.yml` | out of scope for v0; future | -| (no extension) | error: *"schema format unrecognized; please use `.json` or `.toml`"* | +| Extension | Loader | +| --------------- | -------------------------------------------------------------------- | +| `.json` | `serde_json::from_str` | +| `.toml` | `tomljson::from_str` (canonical schema in TOML form) | +| `.yaml`, `.yml` | out of scope for v0; future | +| (no extension) | error: _"schema format unrecognized; please use `.json` or `.toml`"_ | #### `check` standalone (no `mdvs.toml`) -`check` is read-only and stateless, so it can run with `--schema` alone in a directory with no `mdvs.toml`. Defaults applied: +`check` is read-only and stateless, so it can run with `--schema` alone in a +directory with no `mdvs.toml`. Defaults applied: -| Concern | Default when `mdvs.toml` is absent | -|---|---| -| Path | positional argument or `.` | -| File pattern | `**/*.md` | -| Ignore | `.gitignore` + `.mdvsignore` if present | -| `include_bare_files` | `false` | -| Field rules | from `--schema PATH` (mandatory in this mode) | -| Preprocess | from the schema's `x-mdvs.preprocess` (defaults to none) | +| Concern | Default when `mdvs.toml` is absent | +| -------------------- | -------------------------------------------------------- | +| Path | positional argument or `.` | +| File pattern | `**/*.md` | +| Ignore | `.gitignore` + `.mdvsignore` if present | +| `include_bare_files` | `false` | +| Field rules | from `--schema PATH` (mandatory in this mode) | +| Preprocess | from the schema's `x-mdvs.preprocess` (defaults to none) | -`mdvs check ./docs --schema canonical.json` is a complete CI primitive — no `init` step, no project setup. **This is the most important consequence of the simplified design.** It makes mdvs usable as a lightweight markdown linter against a centrally-managed schema. +`mdvs check ./docs --schema canonical.json` is a complete CI primitive — no +`init` step, no project setup. **This is the most important consequence of the +simplified design.** It makes mdvs usable as a lightweight markdown linter +against a centrally-managed schema. #### Why the other commands don't accept `--schema` -| Command | Why no `--schema` | -|---|---| -| `update` | `update` *writes* config. An override would either silently persist (surprising) or silently drop (worse). Inference from filesystem is its single, narrow job. | -| `build` | Builds parquets keyed to a specific config. An ad-hoc schema would diverge from `mdvs.toml`, then the next `build` (without the flag) would re-embed everything. Two ways to set up `.mdvs/` is one too many. | -| `search` | Internally invokes `build` (default `auto_build = true`), so the same argument applies. Search wants to be "this is your project, query it" — not "what would search look like with a different schema?". | -| `info` / `clean` | Don't read schemas. | +| Command | Why no `--schema` | +| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `update` | `update` _writes_ config. An override would either silently persist (surprising) or silently drop (worse). Inference from filesystem is its single, narrow job. | +| `build` | Builds parquets keyed to a specific config. An ad-hoc schema would diverge from `mdvs.toml`, then the next `build` (without the flag) would re-embed everything. Two ways to set up `.mdvs/` is one too many. | +| `search` | Internally invokes `build` (default `auto_build = true`), so the same argument applies. Search wants to be "this is your project, query it" — not "what would search look like with a different schema?". | +| `info` / `clean` | Don't read schemas. | -If a user really wants ad-hoc validation against a schema, that's `check --schema PATH` — purpose-built. They don't need an embedding pipeline for that. +If a user really wants ad-hoc validation against a schema, that's +`check --schema PATH` — purpose-built. They don't need an embedding pipeline for +that. #### Schema validation gate (single, both entry points) -Both `init --schema` and `check --schema` validate the input schema against the same mdvs-compatible subset before using it. The gate is a function `validate_mdvs_schema(schema: &Json) -> Result<(), SchemaError>`: - -1. **Standard JSON Schema keyword allow-list**: `type`, `properties`, `required`, `additionalProperties`, `items`, `enum`, `const`, `minimum`, `maximum`, `exclusiveMinimum`, `exclusiveMaximum`, `minLength`, `maxLength`, `pattern`, `minItems`, `maxItems`, `uniqueItems`, `$schema`, `$id`, `title`, `description`. Plus `x-mdvs`. -2. **`x-mdvs` sub-key allow-list**: `MDVS_KEYS` (settled — `allowed`, `required`, `preprocess`, `definitions`). +Both `init --schema` and `check --schema` validate the input schema against the +same mdvs-compatible subset before using it. The gate is a function +`validate_mdvs_schema(schema: &Json) -> Result<(), SchemaError>`: + +1. **Standard JSON Schema keyword allow-list**: `type`, `properties`, + `required`, `additionalProperties`, `items`, `enum`, `const`, `minimum`, + `maximum`, `exclusiveMinimum`, `exclusiveMaximum`, `minLength`, `maxLength`, + `pattern`, `minItems`, `maxItems`, `uniqueItems`, `$schema`, `$id`, `title`, + `description`. Plus `x-mdvs`. +2. **`x-mdvs` sub-key allow-list**: `MDVS_KEYS` (settled — `allowed`, + `required`, `preprocess`, `definitions`). 3. **Structural constraints**: - Root must be `type: "object"`. - - Properties carrying `x-mdvs.allowed` / `x-mdvs.required` / `x-mdvs.preprocess` must be **directly under the root `properties`** — not nested through composition (`oneOf`/`anyOf`/`allOf`/`not`), conditionals (`if`/`then`/`else`), or `$ref`. -4. **Reject everything else** with a precise path: *"keyword `oneOf` at `properties.author` not supported in mdvs schemas. mdvs accepts a structural subset of JSON Schema 2020-12."* - -Hard-rejected keyword families: `oneOf`, `anyOf`, `allOf`, `not`, `if`/`then`/`else`, `$ref`, `$defs`, `$dynamicRef`, `$dynamicAnchor`, `dependentRequired`, `dependentSchemas`, `patternProperties`, `propertyNames`, `prefixItems`, `contains`, `minContains`, `maxContains`, `unevaluatedProperties`, `unevaluatedItems`, `multipleOf`, `format`, `contentEncoding`, `contentMediaType`, `contentSchema`. - -**Why one gate, not two:** if reference mode (the dropped feature) had passed unrestricted schemas through to `jsonschema::Validator`, mdvs's own semantics around them would have been silently broken (path-scoping wouldn't apply through `oneOf`, `x-mdvs.preprocess` wouldn't fire on properties under `$ref`, etc.). Failing loudly at config load is better than acting "mostly correct." - -The export direction (`mdvs.toml` → JSON Schema) trivially satisfies the gate by construction: the inline DSL's expressivity is a strict subset of the allow-list. + - Properties carrying `x-mdvs.allowed` / `x-mdvs.required` / + `x-mdvs.preprocess` must be **directly under the root `properties`** — not + nested through composition (`oneOf`/`anyOf`/`allOf`/`not`), conditionals + (`if`/`then`/`else`), or `$ref`. +4. **Reject everything else** with a precise path: _"keyword `oneOf` at + `properties.author` not supported in mdvs schemas. mdvs accepts a structural + subset of JSON Schema 2020-12."_ + +Hard-rejected keyword families: `oneOf`, `anyOf`, `allOf`, `not`, +`if`/`then`/`else`, `$ref`, `$defs`, `$dynamicRef`, `$dynamicAnchor`, +`dependentRequired`, `dependentSchemas`, `patternProperties`, `propertyNames`, +`prefixItems`, `contains`, `minContains`, `maxContains`, +`unevaluatedProperties`, `unevaluatedItems`, `multipleOf`, `format`, +`contentEncoding`, `contentMediaType`, `contentSchema`. + +**Why one gate, not two:** if reference mode (the dropped feature) had passed +unrestricted schemas through to `jsonschema::Validator`, mdvs's own semantics +around them would have been silently broken (path-scoping wouldn't apply through +`oneOf`, `x-mdvs.preprocess` wouldn't fire on properties under `$ref`, etc.). +Failing loudly at config load is better than acting "mostly correct." + +The export direction (`mdvs.toml` → JSON Schema) trivially satisfies the gate by +construction: the inline DSL's expressivity is a strict subset of the +allow-list. ## Constraint mapping: `mdvs.toml` ↔ JSON Schema -The translator builds a JSON Schema from `[[fields.field]]` entries. Each field becomes a property in the schema. +The translator builds a JSON Schema from `[[fields.field]]` entries. Each field +becomes a property in the schema. ### Type mapping (Wave C) -| `mdvs.toml` type | JSON Schema | -|---|---| -| `boolean` | `{"type": "boolean"}` | -| `integer` | `{"type": "integer"}` | -| `number` | `{"type": "number"}` | -| `string` | `{"type": "string"}` | -| `array` + `items` | `{"type": "array", "items": }` | +| `mdvs.toml` type | JSON Schema | +| ------------------------------------- | ----------------------------------------- | +| `boolean` | `{"type": "boolean"}` | +| `integer` | `{"type": "integer"}` | +| `number` | `{"type": "number"}` | +| `string` | `{"type": "string"}` | +| `array` + `items` | `{"type": "array", "items": }` | | `object` (flattened via dotted names) | `{"type": "object", "properties": {...}}` | -| Any + `nullable: true` | `{"type": ["", "null"]}` | +| Any + `nullable: true` | `{"type": ["", "null"]}` | ### Constraint mapping -| mdvs constraint | Applies to | JSON Schema keyword(s) | -|---|---|---| -| `categories` | string, integer, array of those | `enum` (scalar), `items.enum` (array) | -| `min` / `max` | integer, number, arrays of those | `minimum`/`maximum` (scalar), `items.minimum`/`items.maximum` (array) | -| `min_length` / `max_length` | string, array | `minLength`/`maxLength` (string), `minItems`/`maxItems` (array) | -| `pattern` | string | `pattern` | +| mdvs constraint | Applies to | JSON Schema keyword(s) | +| --------------------------- | -------------------------------- | --------------------------------------------------------------------- | +| `categories` | string, integer, array of those | `enum` (scalar), `items.enum` (array) | +| `min` / `max` | integer, number, arrays of those | `minimum`/`maximum` (scalar), `items.minimum`/`items.maximum` (array) | +| `min_length` / `max_length` | string, array | `minLength`/`maxLength` (string), `minItems`/`maxItems` (array) | +| `pattern` | string | `pattern` | ### `x-mdvs` extension key (nested object form) -mdvs-specific configuration that JSON Schema doesn't model natively rides on a single nested `x-mdvs` extension key. Both at the schema root (for global config) and on each property (for per-field config). JSON Schema 2020-12 explicitly tolerates unknown keys; standard validators silently ignore the entire `x-mdvs` subtree, while mdvs reads it. +mdvs-specific configuration that JSON Schema doesn't model natively rides on a +single nested `x-mdvs` extension key. Both at the schema root (for global +config) and on each property (for per-field config). JSON Schema 2020-12 +explicitly tolerates unknown keys; standard validators silently ignore the +entire `x-mdvs` subtree, while mdvs reads it. -**Why nested over flat (`x-mdvs-allowed`, `x-mdvs-required`, `x-mdvs-preprocess`)**: one key concentrates all mdvs config, easy to inspect or strip wholesale, less property-level keyword pollution. The flat form's only advantage is matching OpenAPI's `x-amzn-...`, `x-google-...` tradition, which we're not part of. +**Why nested over flat (`x-mdvs-allowed`, `x-mdvs-required`, +`x-mdvs-preprocess`)**: one key concentrates all mdvs config, easy to inspect or +strip wholesale, less property-level keyword pollution. The flat form's only +advantage is matching OpenAPI's `x-amzn-...`, `x-google-...` tradition, which +we're not part of. -**Why `x-mdvs-` and not `x-mdfrontmatter-`**: vendor-extension convention identifies the tool (mdvs) rather than the domain (frontmatter validation). The encoded concepts (`allowed` globs, `preprocess` references) are mdvs-specific in their semantics; another frontmatter validator would name its globs differently. Vendor prefix is honest. +**Why `x-mdvs-` and not `x-mdfrontmatter-`**: vendor-extension convention +identifies the tool (mdvs) rather than the domain (frontmatter validation). The +encoded concepts (`allowed` globs, `preprocess` references) are mdvs-specific in +their semantics; another frontmatter validator would name its globs differently. +Vendor prefix is honest. #### Schema-level `x-mdvs` (global config) @@ -666,11 +814,11 @@ mdvs-specific configuration that JSON Schema doesn't model natively rides on a s } ``` -| Sub-key | What it holds | -|---|---| -| `preprocess.field_names` | Stage 1 preprocessor names, in execution order | -| `preprocess.document` | Stage 3 preprocessor names, in execution order | -| `preprocess.definitions` | Lua scripts referenced by name (`{ name → { type, script } }`). Captures *references*; the .lua files are external. | +| Sub-key | What it holds | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------- | +| `preprocess.field_names` | Stage 1 preprocessor names, in execution order | +| `preprocess.document` | Stage 3 preprocessor names, in execution order | +| `preprocess.definitions` | Lua scripts referenced by name (`{ name → { type, script } }`). Captures _references_; the .lua files are external. | #### Property-level `x-mdvs` (per-field config) @@ -687,58 +835,84 @@ mdvs-specific configuration that JSON Schema doesn't model natively rides on a s } ``` -| Sub-key | What it holds | -|---|---| -| `allowed` | Path-aware glob patterns where this field MAY appear | -| `required` | Path-aware glob patterns where this field MUST appear | +| Sub-key | What it holds | +| ------------ | ------------------------------------------------------------------------------------------ | +| `allowed` | Path-aware glob patterns where this field MAY appear | +| `required` | Path-aware glob patterns where this field MUST appear | | `preprocess` | Stage 2 preprocessor names (built-ins or Lua names from `definitions`), in execution order | #### `mdvs.toml` ↔ `x-mdvs` mapping -| mdvs.toml | JSON Schema | -|---|---| -| `[preprocess]` section keys | `x-mdvs.preprocess.field_names` / `x-mdvs.preprocess.document` | -| `[[preprocess.lua]]` entries | `x-mdvs.preprocess.definitions` | -| `[[fields.field]] preprocess = [...]` | property's `x-mdvs.preprocess` | -| `[[fields.field]] allowed = [...]` | property's `x-mdvs.allowed` | -| `[[fields.field]] required = [...]` | property's `x-mdvs.required` | +| mdvs.toml | JSON Schema | +| ------------------------------------- | -------------------------------------------------------------- | +| `[preprocess]` section keys | `x-mdvs.preprocess.field_names` / `x-mdvs.preprocess.document` | +| `[[preprocess.lua]]` entries | `x-mdvs.preprocess.definitions` | +| `[[fields.field]] preprocess = [...]` | property's `x-mdvs.preprocess` | +| `[[fields.field]] allowed = [...]` | property's `x-mdvs.allowed` | +| `[[fields.field]] required = [...]` | property's `x-mdvs.required` | #### Lua script handling on JSON Schema export / import -- **Export**: Lua entries from `[[preprocess.lua]]` write into `x-mdvs.preprocess.definitions` as `{ type: "lua", script: "" }`. The script *file path* is captured; the script *content* is not inlined. -- **Import** of a JSON Schema referencing Lua scripts: mdvs reads the definitions, expects the script files at the given paths relative to the import target. If a referenced Lua script is missing, mdvs errors at config-load time: *"preprocessor 'normalize_my_tags' referenced but `scripts/normalize_my_tags.lua` not found."* +- **Export**: Lua entries from `[[preprocess.lua]]` write into + `x-mdvs.preprocess.definitions` as `{ type: "lua", script: "" }`. The + script _file path_ is captured; the script _content_ is not inlined. +- **Import** of a JSON Schema referencing Lua scripts: mdvs reads the + definitions, expects the script files at the given paths relative to the + import target. If a referenced Lua script is missing, mdvs errors at + config-load time: _"preprocessor 'normalize_my_tags' referenced but + `scripts/normalize_my_tags.lua` not found."_ -JSON Schema is a structural contract, not a self-contained executable bundle. The script files travel alongside the schema in the user's repository. Same model as a Dockerfile referencing build scripts. +JSON Schema is a structural contract, not a self-contained executable bundle. +The script files travel alongside the schema in the user's repository. Same +model as a Dockerfile referencing build scripts. Standard third-party JSON Schema validators ignore the entire `x-mdvs` subtree. #### Unknown `x-mdvs` keys on import -If a user imports a JSON Schema that contains `x-mdvs` keys mdvs doesn't recognize (typo, or extension introduced in a newer mdvs version), the translator **errors** with the offending key path: +If a user imports a JSON Schema that contains `x-mdvs` keys mdvs doesn't +recognize (typo, or extension introduced in a newer mdvs version), the +translator **errors** with the offending key path: ``` unknown x-mdvs key 'precedence' in property 'title'; either upgrade mdvs to a version that recognizes it, or remove the key ``` -The alternative — silent pass-through — would let typos propagate undetected and round-trip safely through unrelated tooling. Erroring on unknown keys is consistent with serde's `deny_unknown_fields` posture used elsewhere in mdvs's config types. Users who genuinely need forward-compat (rare for mdvs's domain) can hand-edit the file or pin to a specific mdvs version. +The alternative — silent pass-through — would let typos propagate undetected and +round-trip safely through unrelated tooling. Erroring on unknown keys is +consistent with serde's `deny_unknown_fields` posture used elsewhere in mdvs's +config types. Users who genuinely need forward-compat (rare for mdvs's domain) +can hand-edit the file or pin to a specific mdvs version. -The list of recognized keys is maintained as `MDVS_KEYS` (see "Where the translator lives" below). +The list of recognized keys is maintained as `MDVS_KEYS` (see "Where the +translator lives" below). ### Path-scoped validation (`allowed` / `required`) -mdvs's `allowed` and `required` are path-aware globs (per-file-path), not per-value constraints. JSON Schema has no native concept of "this field is required only when the document came from a file matching glob X". The constraint lives entirely in `x-mdvs` and is **enforced by mdvs around `jsonschema::Validator`, not inside it**. +mdvs's `allowed` and `required` are path-aware globs (per-file-path), not +per-value constraints. JSON Schema has no native concept of "this field is +required only when the document came from a file matching glob X". The +constraint lives entirely in `x-mdvs` and is **enforced by mdvs around +`jsonschema::Validator`, not inside it**. #### Runtime model: per-file synthesized schema For each file under validation: -1. **Partition the schema once** at config-load time: strip `x-mdvs.allowed` / `x-mdvs.required` from every property and hold the glob lists aside as a `PathScopeMap { field_name → (allowed_globs, required_globs) }`. The remaining schema is mdvs-agnostic JSON Schema. `jsonschema::Validator::compile` runs once on this stripped form. -2. **Per file, compute the active field set.** Match `file.path` against each field's globs: +1. **Partition the schema once** at config-load time: strip `x-mdvs.allowed` / + `x-mdvs.required` from every property and hold the glob lists aside as a + `PathScopeMap { field_name → (allowed_globs, required_globs) }`. The + remaining schema is mdvs-agnostic JSON Schema. + `jsonschema::Validator::compile` runs once on this stripped form. +2. **Per file, compute the active field set.** Match `file.path` against each + field's globs: - `allowed_globs` matches → field is **permitted** in this file. - `allowed_globs` doesn't match → field is **forbidden** here. - - `required_globs` matches → field is **required** here (additive on top of permitted). -3. **Synthesize a per-file overlay schema** that encodes the active set as standard JSON Schema: + - `required_globs` matches → field is **required** here (additive on top of + permitted). +3. **Synthesize a per-file overlay schema** that encodes the active set as + standard JSON Schema: ```json { @@ -749,8 +923,13 @@ For each file under validation: } ``` - Only properties active for this file appear under `properties`. Forbidden fields are absent → `additionalProperties: false` rejects them if present. -4. **Validate the file's frontmatter twice**: against the once-compiled global schema (covers types, constraints, `enum`, `pattern`, etc.) AND against the per-file overlay (covers presence/absence). Errors from both are merged into the violation report. + Only properties active for this file appear under `properties`. Forbidden + fields are absent → `additionalProperties: false` rejects them if present. + +4. **Validate the file's frontmatter twice**: against the once-compiled global + schema (covers types, constraints, `enum`, `pattern`, etc.) AND against the + per-file overlay (covers presence/absence). Errors from both are merged into + the violation report. #### Worked example @@ -783,105 +962,147 @@ required = ["docs/published/**"] **File `docs/published/launch.md` with `{draft: true}`:** -- Active set: `draft` permitted (matches `docs/**`), required (matches `docs/published/**`). -- Overlay: `{ properties: { draft: { type: boolean } }, required: ["draft"], additionalProperties: false }`. +- Active set: `draft` permitted (matches `docs/**`), required (matches + `docs/published/**`). +- Overlay: + `{ properties: { draft: { type: boolean } }, required: ["draft"], additionalProperties: false }`. - Result: passes. **File `notes/random.md` with `{draft: true}`:** - Active set: `draft` not permitted here. - Overlay: `{ properties: {}, required: [], additionalProperties: false }`. -- Result: jsonschema rejects on `additionalProperties` → `FieldViolation::DisallowedField { field: "draft", file: "notes/random.md" }`. +- Result: jsonschema rejects on `additionalProperties` → + `FieldViolation::DisallowedField { field: "draft", file: "notes/random.md" }`. **File `docs/published/missing.md` with `{}`:** - Active set: `draft` required here. - Overlay: `{ ..., required: ["draft"], ... }`. -- Result: jsonschema rejects on missing required → `FieldViolation::MissingRequired`. +- Result: jsonschema rejects on missing required → + `FieldViolation::MissingRequired`. #### Why per-file synthesis over global `if/then/else` -The alternative — encoding path scoping in vanilla JSON Schema with `if/then/else` keyed on a synthetic `_filepath` property — was rejected for runtime use: - -| Concern | Per-file synthesis (chosen) | Global `if/then/else` | -|---|---|---| -| Schema simplicity | Tiny per-file schemas, mostly cached by active-set hash | N fields × M globs → N×M `allOf/if/then/else` branches | -| Reserved-name pollution | None | Requires reserved `_filepath` property in every document | -| Glob → regex conversion | Done in mdvs's existing `globset` layer | Must convert globs to ECMA-262 regex for `pattern` | -| Fits mdvs's data model | mdvs already has `GlobMap` from inference | Foreign shape, only useful for export | -| Round-trips through tomljson | `x-mdvs` is just nested JSON | Same | - -`_filepath` injection only resurfaces in `mdvs export-schema --portable` (a future, optional flag) where the user wants a self-contained schema for non-mdvs validators that lack `x-mdvs` semantics. Default `export-schema` emits `x-mdvs` intact; downstream non-mdvs tools see a global-scope schema and ignore the extension. This export-portable mode is **out of scope for TODO-0149**; tracked as a follow-up. +The alternative — encoding path scoping in vanilla JSON Schema with +`if/then/else` keyed on a synthetic `_filepath` property — was rejected for +runtime use: + +| Concern | Per-file synthesis (chosen) | Global `if/then/else` | +| ---------------------------- | ------------------------------------------------------- | -------------------------------------------------------- | +| Schema simplicity | Tiny per-file schemas, mostly cached by active-set hash | N fields × M globs → N×M `allOf/if/then/else` branches | +| Reserved-name pollution | None | Requires reserved `_filepath` property in every document | +| Glob → regex conversion | Done in mdvs's existing `globset` layer | Must convert globs to ECMA-262 regex for `pattern` | +| Fits mdvs's data model | mdvs already has `GlobMap` from inference | Foreign shape, only useful for export | +| Round-trips through tomljson | `x-mdvs` is just nested JSON | Same | + +`_filepath` injection only resurfaces in `mdvs export-schema --portable` (a +future, optional flag) where the user wants a self-contained schema for non-mdvs +validators that lack `x-mdvs` semantics. Default `export-schema` emits `x-mdvs` +intact; downstream non-mdvs tools see a global-scope schema and ignore the +extension. This export-portable mode is **out of scope for TODO-0149**; tracked +as a follow-up. #### Inference and `additionalProperties` -The inline DSL has `[fields].ignore` (known fields with no validation) and `[[fields.field]]` (constrained fields). Anything else is "unknown". The per-file overlay's `additionalProperties: false` would reject ignored fields too — which is wrong. +The inline DSL has `[fields].ignore` (known fields with no validation) and +`[[fields.field]]` (constrained fields). Anything else is "unknown". The +per-file overlay's `additionalProperties: false` would reject ignored fields too +— which is wrong. -Resolution: the overlay's `properties` includes both active constrained fields *and* every entry from `[fields].ignore` (translated to property entries with no constraint, e.g. `{}`). Ignored fields permitted everywhere, no globs attached. New / unknown fields (neither constrained nor ignored) hit `additionalProperties: false` → reported as informational `NewField` events, not violations, matching today's behavior. +Resolution: the overlay's `properties` includes both active constrained fields +_and_ every entry from `[fields].ignore` (translated to property entries with no +constraint, e.g. `{}`). Ignored fields permitted everywhere, no globs attached. +New / unknown fields (neither constrained nor ignored) hit +`additionalProperties: false` → reported as informational `NewField` events, not +violations, matching today's behavior. ### Error mapping (`jsonschema::ValidationError` → `ViolationKind`) `jsonschema 0.46` reports errors as `ValidationError` with two relevant fields: -- `instance_path()` — JSON Pointer string identifying the offending value (e.g., `""` for root, `"/draft"` for a property, `"/tags/2"` for an array index). -- `kind()` — `ValidationErrorKind` enum carrying the keyword name and details (limit, expected type, regex pattern, etc.). +- `instance_path()` — JSON Pointer string identifying the offending value (e.g., + `""` for root, `"/draft"` for a property, `"/tags/2"` for an array index). +- `kind()` — `ValidationErrorKind` enum carrying the keyword name and details + (limit, expected type, regex pattern, etc.). -The translator at `crates/mdvs/src/schema/json_schema.rs::map_error()` is a `match` over `ValidationErrorKind` that returns `(ViolationKind, detail)`. Mapping is direct except for one subtlety on `Type`: when the actual instance value is `null`, the error is routed to `NullNotAllowed` (semantic mdvs concept) rather than `WrongType` — `nullable: false` translates to a non-null type in JSON Schema, so a null value triggers a generic `Type` error that we re-classify. +The translator at `crates/mdvs/src/schema/json_schema.rs::map_error()` is a +`match` over `ValidationErrorKind` that returns `(ViolationKind, detail)`. +Mapping is direct except for one subtlety on `Type`: when the actual instance +value is `null`, the error is routed to `NullNotAllowed` (semantic mdvs concept) +rather than `WrongType` — `nullable: false` translates to a non-null type in +JSON Schema, so a null value triggers a generic `Type` error that we +re-classify. #### Mapping table -| `jsonschema::ErrorKind` | `mdvs::ViolationKind` | Notes | -|---|---|---| -| `Required { property }` | `MissingRequired` | property name → `field` | -| `AdditionalProperties { unexpected }` | `Disallowed` (or `NewField` informational, see "Inference and additionalProperties") | each name in `unexpected` becomes one violation | -| `Type { kind }` (instance is null) | `NullNotAllowed` | re-classified — JSON Schema has no native "non-null required" | -| `Type { kind }` (instance is not null) | `WrongType` | detail: `expected {kind:?}, got {actual_type}` | -| `Enum { options }` | `InvalidCategory` | covers TODO-0006 enum constraint | -| `Constant { expected_value }` | `InvalidCategory` | const = single-element enum semantically | -| `Minimum { limit }` | `OutOfRange` | TODO-0008 | -| `Maximum { limit }` | `OutOfRange` | TODO-0008 | -| `ExclusiveMinimum { limit }` | `OutOfRange` | TODO-0008 | -| `ExclusiveMaximum { limit }` | `OutOfRange` | TODO-0008 | -| `MultipleOf { multiple_of }` | `OutOfRange` | not currently in mdvs DSL but emitted if user writes JSON Schema directly | -| `MinLength { limit }` | `OutOfRange` | TODO-0010 | -| `MaxLength { limit }` | `OutOfRange` | TODO-0010 | -| `Pattern { pattern }` | `WrongType` | TODO-0145; "value doesn't match expected shape" | -| `MinItems { limit }` | `OutOfRange` | array bounds | -| `MaxItems { limit }` | `OutOfRange` | array bounds | -| `UniqueItems` | `OutOfRange` | array bounds | -| any other variant | `WrongType` (fallback) with keyword name in `detail` | covers user-imported schemas with keywords mdvs doesn't model (`oneOf`, `if/then/else`, `dependentRequired`, etc.) | +| `jsonschema::ErrorKind` | `mdvs::ViolationKind` | Notes | +| -------------------------------------- | ------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------ | +| `Required { property }` | `MissingRequired` | property name → `field` | +| `AdditionalProperties { unexpected }` | `Disallowed` (or `NewField` informational, see "Inference and additionalProperties") | each name in `unexpected` becomes one violation | +| `Type { kind }` (instance is null) | `NullNotAllowed` | re-classified — JSON Schema has no native "non-null required" | +| `Type { kind }` (instance is not null) | `WrongType` | detail: `expected {kind:?}, got {actual_type}` | +| `Enum { options }` | `InvalidCategory` | covers TODO-0006 enum constraint | +| `Constant { expected_value }` | `InvalidCategory` | const = single-element enum semantically | +| `Minimum { limit }` | `OutOfRange` | TODO-0008 | +| `Maximum { limit }` | `OutOfRange` | TODO-0008 | +| `ExclusiveMinimum { limit }` | `OutOfRange` | TODO-0008 | +| `ExclusiveMaximum { limit }` | `OutOfRange` | TODO-0008 | +| `MultipleOf { multiple_of }` | `OutOfRange` | not currently in mdvs DSL but emitted if user writes JSON Schema directly | +| `MinLength { limit }` | `OutOfRange` | TODO-0010 | +| `MaxLength { limit }` | `OutOfRange` | TODO-0010 | +| `Pattern { pattern }` | `WrongType` | TODO-0145; "value doesn't match expected shape" | +| `MinItems { limit }` | `OutOfRange` | array bounds | +| `MaxItems { limit }` | `OutOfRange` | array bounds | +| `UniqueItems` | `OutOfRange` | array bounds | +| any other variant | `WrongType` (fallback) with keyword name in `detail` | covers user-imported schemas with keywords mdvs doesn't model (`oneOf`, `if/then/else`, `dependentRequired`, etc.) | #### Field-name extraction -The `field` slot of `FieldViolation` is the **top-level frontmatter field name**, derived from `instance_path()` by taking the first JSON Pointer segment: +The `field` slot of `FieldViolation` is the **top-level frontmatter field +name**, derived from `instance_path()` by taking the first JSON Pointer segment: -- `""` (empty path, root) — used by `Required` and `AdditionalProperties` errors. Field name comes from the `kind` payload (`property` or each entry of `unexpected`), not the path. +- `""` (empty path, root) — used by `Required` and `AdditionalProperties` + errors. Field name comes from the `kind` payload (`property` or each entry of + `unexpected`), not the path. - `"/draft"` → `field = "draft"`. - `"/tags/2"` → `field = "tags"`, `detail` annotates the index (`"item 2"`). -- `"/address/city"` (Wave C, after object flattening) → `field = "address.city"` (the flattened key already encodes the nested shape; no further extraction needed). +- `"/address/city"` (Wave C, after object flattening) → `field = "address.city"` + (the flattened key already encodes the nested shape; no further extraction + needed). #### Detail rendering -`ViolatingFile.detail` is populated from the `kind` payload to preserve the diagnostic information that the bare `ViolationKind` enum drops: +`ViolatingFile.detail` is populated from the `kind` payload to preserve the +diagnostic information that the bare `ViolationKind` enum drops: -| Kind | Detail | -|---|---| -| `Type { kind }` (non-null) | `"expected {kind}, got {actual_type}"` (e.g., `"expected boolean, got string"`) | -| `Type { kind }` (null) | `None` — `NullNotAllowed` is self-explanatory | -| `Minimum/Maximum { limit }` | `"value {value} {< | >} {limit}"` | -| `MinLength/MaxLength { limit }` | `"length {len} {< | >} {limit}"` | -| `Pattern { pattern }` | `"value does not match /{pattern}/"` | -| `Enum { options }` | `"got {value}, expected one of {options}"` (truncated if >5 options) | -| `MinItems/MaxItems { limit }` | `"array length {n} {< | >} {limit}"` | -| `UniqueItems` | `"duplicate items at indices {a}, {b}"` | +| Kind | Detail | +| ------------------------------- | ------------------------------------------------------------------------------- | +| `Type { kind }` (non-null) | `"expected {kind}, got {actual_type}"` (e.g., `"expected boolean, got string"`) | +| `Type { kind }` (null) | `None` — `NullNotAllowed` is self-explanatory | +| `Minimum/Maximum { limit }` | `"value {value} {< \| >} {limit}"` | +| `MinLength/MaxLength { limit }` | `"length {len} {< \| >} {limit}"` | +| `Pattern { pattern }` | `"value does not match /{pattern}/"` | +| `Enum { options }` | `"got {value}, expected one of {options}"` (truncated if >5 options) | +| `MinItems/MaxItems { limit }` | `"array length {n} {< \| >} {limit}"` | +| `UniqueItems` | `"duplicate items at indices {a}, {b}"` | #### Validated by -`scripts/test_violation_mapping.rs` — 22 cases exercising every keyword the translator emits, plus `Constant` and `MultipleOf` (which mdvs doesn't generate today but may receive from user-imported schemas). All `ErrorKind` variants observed match the table above. The null-vs-WrongType distinction is exercised at test 5 (`null` against `type: "string"` → `NullNotAllowed`) and test 6 (`null` against `type: ["string", "null"]` → no error, sanity for `nullable: true`). +`scripts/test_violation_mapping.rs` — 22 cases exercising every keyword the +translator emits, plus `Constant` and `MultipleOf` (which mdvs doesn't generate +today but may receive from user-imported schemas). All `ErrorKind` variants +observed match the table above. The null-vs-WrongType distinction is exercised +at test 5 (`null` against `type: "string"` → `NullNotAllowed`) and test 6 +(`null` against `type: ["string", "null"]` → no error, sanity for +`nullable: true`). ### Where the translator lives -The `mdvs.toml` DSL ↔ canonical JSON Schema translation is mdvs domain logic and lives in **`crates/mdvs/src/schema/json_schema.rs`**, *not* in `tomljson`. This boundary is structural: `tomljson` must remain a generic, mdvs-agnostic crate to be publishable and reusable. +The `mdvs.toml` DSL ↔ canonical JSON Schema translation is mdvs domain logic +and lives in **`crates/mdvs/src/schema/json_schema.rs`**, _not_ in `tomljson`. +This boundary is structural: `tomljson` must remain a generic, mdvs-agnostic +crate to be publishable and reusable. ``` ┌────────────────────────────────────────────────────────────────┐ @@ -932,21 +1153,24 @@ The `mdvs.toml` DSL ↔ canonical JSON Schema translation is mdvs domain logic a ### When each crate is involved (per command) -| Command | Reads | `tomljson` involved? | -|---|---|---| -| `check [path]` (no flag) | `mdvs.toml` (DSL) | No | -| `check [path] --schema schema.json` | JSON file | No (input is already JSON) | +| Command | Reads | `tomljson` involved? | +| ----------------------------------- | --------------------- | -------------------------------------------- | +| `check [path]` (no flag) | `mdvs.toml` (DSL) | No | +| `check [path] --schema schema.json` | JSON file | No (input is already JSON) | | `check [path] --schema schema.toml` | canonical TOML schema | **Yes** — TOML→JSON via `tomljson::from_str` | -| `build [path]` | `mdvs.toml` | No (calls `check` internally) | -| `update [path]` | `mdvs.toml` | No | -| `init [path]` | scans .md files | No | -| `init [path] --schema schema.json` | JSON file | No (input is already JSON) | -| `init [path] --schema schema.toml` | canonical TOML schema | **Yes** — TOML→JSON via `tomljson::from_str` | -| `export-schema --format json` | `mdvs.toml` | No (output is JSON) | -| `export-schema --format toml` | `mdvs.toml` | **Yes** — output is JSON-shaped data as TOML | -| `search`, `info`, `clean` | various | No | - -`tomljson` is only invoked at the boundary where mdvs reads or writes a *JSON-shaped schema document* in TOML form. mdvs's own config (`mdvs.toml`) uses the high-level `toml` crate (typed serde, where the TOML is structurally a typed config, not a JSON Schema). +| `build [path]` | `mdvs.toml` | No (calls `check` internally) | +| `update [path]` | `mdvs.toml` | No | +| `init [path]` | scans .md files | No | +| `init [path] --schema schema.json` | JSON file | No (input is already JSON) | +| `init [path] --schema schema.toml` | canonical TOML schema | **Yes** — TOML→JSON via `tomljson::from_str` | +| `export-schema --format json` | `mdvs.toml` | No (output is JSON) | +| `export-schema --format toml` | `mdvs.toml` | **Yes** — output is JSON-shaped data as TOML | +| `search`, `info`, `clean` | various | No | + +`tomljson` is only invoked at the boundary where mdvs reads or writes a +_JSON-shaped schema document_ in TOML form. mdvs's own config (`mdvs.toml`) uses +the high-level `toml` crate (typed serde, where the TOML is structurally a typed +config, not a JSON Schema). ### Translator implementation sketch @@ -1005,54 +1229,90 @@ pub fn canonical_to_dsl(schema: &Value) -> Result { } ``` -The `MDVS_KEYS` constant is the single source of truth for which top-level field names get gathered into `x-mdvs`. Adding a new mdvs-specific field (in some future TODO) means: (1) add the field to `MdvsToml`, (2) add its name to `MDVS_KEYS`, (3) update `field_to_property` and `canonical_to_dsl` accordingly. Mechanical, well-localized. +The `MDVS_KEYS` constant is the single source of truth for which top-level field +names get gathered into `x-mdvs`. Adding a new mdvs-specific field (in some +future TODO) means: (1) add the field to `MdvsToml`, (2) add its name to +`MDVS_KEYS`, (3) update `field_to_property` and `canonical_to_dsl` accordingly. +Mechanical, well-localized. ### tomljson testability isolation -Because `tomljson`'s API is ` ↔ `, none of its tests reference mdvs concepts. The 37 + 18 + 31 prototype cases all express their fixtures as `serde_json::Value` literals (via `json!(...)` macros) — `MdvsToml`, `x-mdvs`, `[[fields.field]]` are nowhere in the prototype scripts. That's the correct shape: it confirms `tomljson` doesn't leak mdvs assumptions and would be useful to anyone needing lossless JSON↔TOML translation. +Because `tomljson`'s API is ` ↔ `, none of its tests +reference mdvs concepts. The 37 + 18 + 31 prototype cases all express their +fixtures as `serde_json::Value` literals (via `json!(...)` macros) — `MdvsToml`, +`x-mdvs`, `[[fields.field]]` are nowhere in the prototype scripts. That's the +correct shape: it confirms `tomljson` doesn't leak mdvs assumptions and would be +useful to anyone needing lossless JSON↔TOML translation. ## `tomljson` crate scope (Wave A) -A standalone crate providing **lossless bidirectional translation between TOML documents and JSON-shaped data** (`serde_json::Value`). Pure translation, no validation, no schema awareness. +A standalone crate providing **lossless bidirectional translation between TOML +documents and JSON-shaped data** (`serde_json::Value`). Pure translation, no +validation, no schema awareness. ### Scope -- **In scope**: any JSON value ↔ any TOML value, with explicit handling of the impedance mismatches (null, top-level non-table, integer range, datetime asymmetry). -- **Motivating use case**: serializing JSON Schema 2020-12 documents as TOML — every keyword roundtrips losslessly because the underlying translator is exhaustive. -- **Distinguishes from `jsontoml` / `toml2json`** (existing CLI tools): we are a Rust *library* with bidirectional API and impedance handling, not a one-way command-line filter. +- **In scope**: any JSON value ↔ any TOML value, with explicit handling of the + impedance mismatches (null, top-level non-table, integer range, datetime + asymmetry). +- **Motivating use case**: serializing JSON Schema 2020-12 documents as TOML — + every keyword roundtrips losslessly because the underlying translator is + exhaustive. +- **Distinguishes from `jsontoml` / `toml2json`** (existing CLI tools): we are a + Rust _library_ with bidirectional API and impedance handling, not a one-way + command-line filter. ### Out of scope - Schema validation (delegated to the `jsonschema` crate). - A schema language for TOML files (Tombi/Taplo's domain — different problem). -- Any JSON Schema-aware behavior — the crate doesn't know what `properties` or `oneOf` mean. It treats them as opaque keys. -- Format conversions other than JSON↔TOML (YAML, CBOR, MessagePack — remarshal's domain). +- Any JSON Schema-aware behavior — the crate doesn't know what `properties` or + `oneOf` mean. It treats them as opaque keys. +- Format conversions other than JSON↔TOML (YAML, CBOR, MessagePack — + remarshal's domain). ### How JSON Schema "rides" the translator -JSON Schema 2020-12 is just JSON-shaped data. Every keyword (`oneOf`, `anyOf`, `$ref`, `$defs`, `if`/`then`/`else`, `const`, `enum`, `pattern`, etc.) is a regular JSON value at some path. Because the translator is exhaustive over JSON, it roundtrips every JSON Schema document. The encoding *conventions* below describe how common JSON Schema constructs render in TOML — but they're emergent, not coded into the crate: +JSON Schema 2020-12 is just JSON-shaped data. Every keyword (`oneOf`, `anyOf`, +`$ref`, `$defs`, `if`/`then`/`else`, `const`, `enum`, `pattern`, etc.) is a +regular JSON value at some path. Because the translator is exhaustive over JSON, +it roundtrips every JSON Schema document. The encoding _conventions_ below +describe how common JSON Schema constructs render in TOML — but they're +emergent, not coded into the crate: - JSON object keys → TOML keys verbatim (`minLength`, `pattern`, `enum`). - JSON arrays of objects → TOML arrays of tables (`[[oneOf]]`). -- JSON keys with `$` or special chars → TOML quoted keys (`"$ref" = "..."`, `["$defs".address]`). -- Nested JSON objects → nested TOML tables (`[properties.user.properties.email]`). +- JSON keys with `$` or special chars → TOML quoted keys (`"$ref" = "..."`, + `["$defs".address]`). +- Nested JSON objects → nested TOML tables + (`[properties.user.properties.email]`). ### Implementation strategy -Use **`toml_writer` (low-level emitter) for serialization** and **`toml::from_str` (parser) + tree walk for deserialization**. The crate is a thin dispatcher around both. +Use **`toml_writer` (low-level emitter) for serialization** and +**`toml::from_str` (parser) + tree walk for deserialization**. The crate is a +thin dispatcher around both. This was settled after surveying the toml-rs crate family: -| Approach considered | Outcome | -|---|---| -| Wrap `toml::Serializer` at the serde trait level | **Infeasible.** `toml`'s internal `MapValueSerializer`, `SerializeDocumentTable`, etc. are `pub(crate)` — only the outer `Serializer` is wrappable, which doesn't intercept any nested values. | -| Wrap `toml_edit::Serializer` (its internal types are public) | Feasible but ~600 LOC of mechanical trait forwarding, plus a heavier dependency. | -| Pivot through `serde_json::Value` for both directions | Works (~250 LOC) but allocates an intermediate Value tree per call. | -| **Use `toml_writer` directly for serialize, `toml::from_str` + walk for deserialize** | **Chosen.** ~400 LOC, zero allocation on serialize, no state-machine wrapping required. Spike validated in `scripts/test_tomljson_writer.rs`. | - -`toml_writer` is the toml-rs project's low-level emitter crate (sibling to `toml`, `toml_edit`, `toml_parser`, `toml_datetime`). It exposes primitives like `open_table_header`, `key`, `keyval_sep`, `value`, `open_array`, `val_sep`, etc., letting us drive TOML emission ourselves while delegating syntactic correctness (string escaping, float formatting, key quoting, multi-line strings) to the upstream crate. - -The dispatcher walks a `serde_json::Value` and calls `toml_writer` primitives. When it sees a JSON value TOML can't represent natively (`null`, `u64 > i64::MAX`, top-level non-table), it applies the placeholder/wrapper/error strategy itself. +| Approach considered | Outcome | +| ------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Wrap `toml::Serializer` at the serde trait level | **Infeasible.** `toml`'s internal `MapValueSerializer`, `SerializeDocumentTable`, etc. are `pub(crate)` — only the outer `Serializer` is wrappable, which doesn't intercept any nested values. | +| Wrap `toml_edit::Serializer` (its internal types are public) | Feasible but ~600 LOC of mechanical trait forwarding, plus a heavier dependency. | +| Pivot through `serde_json::Value` for both directions | Works (~250 LOC) but allocates an intermediate Value tree per call. | +| **Use `toml_writer` directly for serialize, `toml::from_str` + walk for deserialize** | **Chosen.** ~400 LOC, zero allocation on serialize, no state-machine wrapping required. Spike validated in `scripts/test_tomljson_writer.rs`. | + +`toml_writer` is the toml-rs project's low-level emitter crate (sibling to +`toml`, `toml_edit`, `toml_parser`, `toml_datetime`). It exposes primitives like +`open_table_header`, `key`, `keyval_sep`, `value`, `open_array`, `val_sep`, +etc., letting us drive TOML emission ourselves while delegating syntactic +correctness (string escaping, float formatting, key quoting, multi-line strings) +to the upstream crate. + +The dispatcher walks a `serde_json::Value` and calls `toml_writer` primitives. +When it sees a JSON value TOML can't represent natively (`null`, +`u64 > i64::MAX`, top-level non-table), it applies the placeholder/wrapper/error +strategy itself. ### API surface (sketch) @@ -1083,31 +1343,53 @@ tomljson::from_str(s: &str) -> Result; tomljson::from_str_with_options(s: &str, opts: &TomlJsonOptions) -> Result; ``` -The crate accepts and produces `serde_json::Value` directly. Users with typed Rust structs convert via `serde_json::to_value` / `serde_json::from_value` at the boundary. A generic `Serialize`/`DeserializeOwned` wrapper is a v0.2 ergonomic enhancement, not in scope for v0.1. +The crate accepts and produces `serde_json::Value` directly. Users with typed +Rust structs convert via `serde_json::to_value` / `serde_json::from_value` at +the boundary. A generic `Serialize`/`DeserializeOwned` wrapper is a v0.2 +ergonomic enhancement, not in scope for v0.1. ### No standalone / self-describing format -An earlier design proposed a "standalone" API where a TOML document declared its own placeholder via a `$tomljson-null` root directive (or, alternatively, a magic comment header), so receiving tools could decode without prior knowledge of the placeholder. **This was rejected:** +An earlier design proposed a "standalone" API where a TOML document declared its +own placeholder via a `$tomljson-null` root directive (or, alternatively, a +magic comment header), so receiving tools could decode without prior knowledge +of the placeholder. **This was rejected:** -- An in-band directive key (`$tomljson-null` at root) collides with arbitrary JSON data — non–JSON-Schema documents can legitimately have a top-level key with that name. The encoder would emit duplicate keys; the decoder would silently strip user data. -- A magic comment header invents a non-standard convention with no precedent in TOML, and depends on raw-string preprocessing fragile to formatters. +- An in-band directive key (`$tomljson-null` at root) collides with arbitrary + JSON data — non–JSON-Schema documents can legitimately have a top-level key + with that name. The encoder would emit duplicate keys; the decoder would + silently strip user data. +- A magic comment header invents a non-standard convention with no precedent in + TOML, and depends on raw-string preprocessing fragile to formatters. -For v0.1, callers always supply `TomlJsonOptions` (or accept the default placeholder). When sharing TOML files between tools, the placeholder convention is documented out-of-band — in the project's docs, a sibling config, or a fixed convention. If a concrete need for self-describing TOML emerges later, we add it then with whatever convention proves necessary, informed by real consumers. +For v0.1, callers always supply `TomlJsonOptions` (or accept the default +placeholder). When sharing TOML files between tools, the placeholder convention +is documented out-of-band — in the project's docs, a sibling config, or a fixed +convention. If a concrete need for self-describing TOML emerges later, we add it +then with whatever convention proves necessary, informed by real consumers. ### Documentation requirement The crate's `README.md` MUST prominently document: -- The full per-direction policy table (encode-side and decode-side handling of every impedance gap). -- Why `+inf` / `-inf` / `NaN` error on decode despite storage layers supporting them. -- The intentional asymmetry: TOML grammar has no "key with null value" form, so absent-key and present-with-null must be distinguished by the placeholder convention. +- The full per-direction policy table (encode-side and decode-side handling of + every impedance gap). +- Why `+inf` / `-inf` / `NaN` error on decode despite storage layers supporting + them. +- The intentional asymmetry: TOML grammar has no "key with null value" form, so + absent-key and present-with-null must be distinguished by the placeholder + convention. - Why standalone / self-describing TOML is out of scope for v0.1. -These are first-class design constraints, not footnotes. Users picking up the crate should understand them before integrating. +These are first-class design constraints, not footnotes. Users picking up the +crate should understand them before integrating. ### Null encoding -TOML has no `null`. The crate represents JSON `null` as a **user-chosen string placeholder**, default `"__null__"`. The placeholder occupies the slot anywhere a null appears — top-level, nested, or inside arrays — keeping arrays homogeneous-looking. Examples: +TOML has no `null`. The crate represents JSON `null` as a **user-chosen string +placeholder**, default `"__null__"`. The placeholder occupies the slot anywhere +a null appears — top-level, nested, or inside arrays — keeping arrays +homogeneous-looking. Examples: ```toml default = "__null__" @@ -1115,25 +1397,41 @@ const = "__null__" enum = [1, 2, "__null__"] ``` -The decoder (with the placeholder string in hand) walks the parsed TOML and replaces any string equal to the placeholder with JSON `null`. +The decoder (with the placeholder string in hand) walks the parsed TOML and +replaces any string equal to the placeholder with JSON `null`. -**Custom placeholder** — callers pass `TomlJsonOptions::null_placeholder` to encode/decode functions. The placeholder is supplied by the application out-of-band (see "No standalone / self-describing format" above for why). For mdvs, the placeholder is a constant the crate uses internally; it never appears as a directive in `mdvs.toml`. +**Custom placeholder** — callers pass `TomlJsonOptions::null_placeholder` to +encode/decode functions. The placeholder is supplied by the application +out-of-band (see "No standalone / self-describing format" above for why). For +mdvs, the placeholder is a constant the crate uses internally; it never appears +as a directive in `mdvs.toml`. -**Collision behavior** — error. If a real string value in the input equals the placeholder, the encoder fails with a clear message asking the user to pass a different `null_placeholder` via `TomlJsonOptions`. +**Collision behavior** — error. If a real string value in the input equals the +placeholder, the encoder fails with a clear message asking the user to pass a +different `null_placeholder` via `TomlJsonOptions`. ### Boolean-schema root encoding -JSON Schema 2020-12 allows the literal `true` or `false` as a complete schema (always-valid / always-invalid). TOML's root must be a table, so the crate wraps non-object roots under a single key `__root__`: +JSON Schema 2020-12 allows the literal `true` or `false` as a complete schema +(always-valid / always-invalid). TOML's root must be a table, so the crate wraps +non-object roots under a single key `__root__`: ```toml __root__ = true ``` -The decoder unwraps a root table that has exactly one entry named `__root__`. The dunder-style name follows the same convention as `__null__` to make sentinels visually distinct from real keys. Collision behavior matches null: error if a real top-level key is named `__root__`. +The decoder unwraps a root table that has exactly one entry named `__root__`. +The dunder-style name follows the same convention as `__null__` to make +sentinels visually distinct from real keys. Collision behavior matches null: +error if a real top-level key is named `__root__`. ### Integer range — strict spec adherence -TOML integers are 64-bit signed (−2⁶³ to 2⁶³−1). The TOML 1.1 spec mandates: *"if an integer cannot be represented losslessly, an error must be thrown."* JSON allows arbitrary-precision integers; `serde_json::Number` admits values up to `u64::MAX`. There is exactly one bit of mismatch: the range `(i64::MAX, u64::MAX]`. +TOML integers are 64-bit signed (−2⁶³ to 2⁶³−1). The TOML 1.1 spec mandates: +_"if an integer cannot be represented losslessly, an error must be thrown."_ +JSON allows arbitrary-precision integers; `serde_json::Number` admits values up +to `u64::MAX`. There is exactly one bit of mismatch: the range +`(i64::MAX, u64::MAX]`. The crate adheres to the TOML spec — values in that range error on encode: @@ -1143,10 +1441,14 @@ input contains integer 18446744073709551615 which exceeds TOML's signed 64-bit r ``` We considered (and rejected) two alternatives: -- **Stringified bigint with directive prefix** (mirroring the null placeholder) — works, but no other TOML tool would understand it; portability lost. + +- **Stringified bigint with directive prefix** (mirroring the null placeholder) + — works, but no other TOML tool would understand it; portability lost. - **Silent f64 fallback** — corrupts the value above 2⁵³; rejected outright. -Real-world JSON data containing values > i64::MAX is rare (snowflake IDs, cryptographic nonces). When they do appear, an explicit error beats silent approximation. +Real-world JSON data containing values > i64::MAX is rare (snowflake IDs, +cryptographic nonces). When they do appear, an explicit error beats silent +approximation. ### Per-direction policy table @@ -1154,98 +1456,173 @@ Bidirectional translation involves asymmetric concerns. The full policy: #### JSON → TOML (encode) -| Concern | Handling | -|---|---| -| `Json::Null` value (anywhere) | Substitute the placeholder string (default `"__null__"`) | -| Top-level non-table value (bool, scalar, array) | Wrap under `__root__` key | -| `serde_json::Number` representable as `u64 > i64::MAX` | **Error** — TOML's signed 64-bit limit; JSON spec is wider | -| String value equal to the configured null placeholder | **Error** — collision; caller must pass a different `null_placeholder` via `TomlJsonOptions` | -| Strings that look like TOML literals (`"42"`, `"2026-05-04"`, `"true"`, `"inf"`) | Always quoted on output (`toml_writer` handles escaping) | -| Other JSON values (string, finite number, bool, array, object) | Direct emission via `toml_writer` primitives | +| Concern | Handling | +| -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | +| `Json::Null` value (anywhere) | Substitute the placeholder string (default `"__null__"`) | +| Top-level non-table value (bool, scalar, array) | Wrap under `__root__` key | +| `serde_json::Number` representable as `u64 > i64::MAX` | **Error** — TOML's signed 64-bit limit; JSON spec is wider | +| String value equal to the configured null placeholder | **Error** — collision; caller must pass a different `null_placeholder` via `TomlJsonOptions` | +| Strings that look like TOML literals (`"42"`, `"2026-05-04"`, `"true"`, `"inf"`) | Always quoted on output (`toml_writer` handles escaping) | +| Other JSON values (string, finite number, bool, array, object) | Direct emission via `toml_writer` primitives | #### TOML → JSON (decode) -| Concern | Handling | -|---|---| -| TOML string equal to the configured placeholder | Decode as `Json::Null` | -| Root-level table containing exactly one key `__root__` | Unwrap and return the inner value | -| TOML datetime (any of: `Date`, `Time`, `LocalDateTime`, `OffsetDateTime`) | Decode as `Json::String` using the canonical RFC 3339 form (`"2026-05-04"`, `"09:30:00"`, `"2026-05-04T09:30:00Z"`, etc.) | -| TOML float `+inf` / `-inf` | **Error** — invalid JSON; users wanting "no upper/lower bound" should omit the keyword instead | -| TOML float `NaN` | **Error** — invalid JSON; in JSON Schema, "no constraint" is expressed by absence, not NaN | -| TOML integer (i64) | Decode as `Json::Number` (i64-backed) | -| TOML finite float (f64) | Decode as `Json::Number` (f64-backed) | -| TOML bool, array, table | Recursively decode | -| Absent TOML key | Absent in JSON object — no special handling. **The TOML grammar has no syntax for "key with no value"; that's a parse error, not a null.** Distinct from JSON's `{ "key": null }` (key present, value null), which round-trips through the placeholder. | - -The TOML→JSON direction is the **dominant one** in mdvs: users author JSON Schema in TOML form, mdvs decodes it for internal validation. Both spike scripts validate this direction; `scripts/test_tomljson_decode.rs` covers it explicitly with 31 cases. +| Concern | Handling | +| ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| TOML string equal to the configured placeholder | Decode as `Json::Null` | +| Root-level table containing exactly one key `__root__` | Unwrap and return the inner value | +| TOML datetime (any of: `Date`, `Time`, `LocalDateTime`, `OffsetDateTime`) | Decode as `Json::String` using the canonical RFC 3339 form (`"2026-05-04"`, `"09:30:00"`, `"2026-05-04T09:30:00Z"`, etc.) | +| TOML float `+inf` / `-inf` | **Error** — invalid JSON; users wanting "no upper/lower bound" should omit the keyword instead | +| TOML float `NaN` | **Error** — invalid JSON; in JSON Schema, "no constraint" is expressed by absence, not NaN | +| TOML integer (i64) | Decode as `Json::Number` (i64-backed) | +| TOML finite float (f64) | Decode as `Json::Number` (f64-backed) | +| TOML bool, array, table | Recursively decode | +| Absent TOML key | Absent in JSON object — no special handling. **The TOML grammar has no syntax for "key with no value"; that's a parse error, not a null.** Distinct from JSON's `{ "key": null }` (key present, value null), which round-trips through the placeholder. | + +The TOML→JSON direction is the **dominant one** in mdvs: users author JSON +Schema in TOML form, mdvs decodes it for internal validation. Both spike scripts +validate this direction; `scripts/test_tomljson_decode.rs` covers it explicitly +with 31 cases. ### Why we error on `+inf` / `-inf` / `NaN` These values **could** survive in storage layers we use: + - Apache Parquet stores Float64 as IEEE 754, including non-finite values. - Apache Arrow Float64 supports them in memory. -- LanceDB Float64 supports them (recent fix per [lancedb#3153](https://github.com/lancedb/lancedb/issues/3153)). +- LanceDB Float64 supports them (recent fix per + [lancedb#3153](https://github.com/lancedb/lancedb/issues/3153)). But they **cannot** survive validation: -- `serde_json::Number::from_f64(f64::NAN)` returns `None`. Documented behavior: *"Infinite or NaN values are not JSON numbers."* -- The `jsonschema` crate operates on `serde_json::Value`, so it can never see them. -- JSON Schema itself has no syntax for "no upper bound" except omitting `maximum`. -So even if `tomljson` silently allowed `+inf`, the validation pipeline would refuse it later with a worse error. Erroring at the TOML→JSON boundary surfaces the problem immediately and points users at the idiomatic JSON Schema solution (omit the keyword). +- `serde_json::Number::from_f64(f64::NAN)` returns `None`. Documented behavior: + _"Infinite or NaN values are not JSON numbers."_ +- The `jsonschema` crate operates on `serde_json::Value`, so it can never see + them. +- JSON Schema itself has no syntax for "no upper bound" except omitting + `maximum`. + +So even if `tomljson` silently allowed `+inf`, the validation pipeline would +refuse it later with a worse error. Erroring at the TOML→JSON boundary surfaces +the problem immediately and points users at the idiomatic JSON Schema solution +(omit the keyword). ### Other JSON↔TOML edge cases (handled) -| Case | Behavior | -|---|---| -| Strings shaped like TOML literals (`"true"`, `"42"`, `"2026-05-04"`, `"inf"`) | Encoder always quotes; decoder parses as strings. No collision. | -| Hand-written unquoted TOML datetime (`default = 2026-05-04`) | Decoder converts `Toml::Datetime` to a JSON string. TOML form is normalized to quoted on re-encode. | -| Hand-written unquoted TOML local time (`default = 09:30:00`) | Same as datetime — converted to JSON string `"09:30:00"`. | -| Empty root schema `{}` | Encodes to empty TOML, decodes back to `{}`. | -| Heterogeneous arrays | TOML 1.1 spec explicitly allows `numbers = [0.1, 0.2, 1, 2]`. The `toml` crate honors this. | -| `serde_json` u64 within i64 range | Roundtrips losslessly via `Toml::Integer`. | -| f64 precision (subnormals, `f64::MIN_POSITIVE`, `f64::MAX`, `0.1 + 0.2`) | Rust `toml` crate uses Ryū-style shortest-roundtrip serialization — values roundtrip exactly. Concern raised in [toml-lang/toml#44](https://github.com/toml-lang/toml/issues/44) does not apply in practice. | -| Unicode in strings (`"café ☕ — 日本語 — 🚀"`) | Roundtrips losslessly. | -| Empty string values (`""`) | Roundtrip. | -| Strings with embedded newlines | Encoded as TOML multi-line basic strings (`"""..."""`); decoded back to JSON strings with `\n`. | -| TOML `+inf`, `-inf`, `nan` floats on decode | **Error.** See "Per-direction policy table" above — `serde_json::Number::from_f64` rejects them, and `jsonschema` never sees them, so erroring early is the only honest behavior. | -| TOML datetime on decode (any of 4 variants) | **JSON string** in canonical RFC 3339 form. JSON Schema uses strings + `format` for dates. | -| Absent TOML key vs. JSON `null` | TOML grammar has no syntax for "key with no value" (parse error). Absent TOML key → absent JSON object key. JSON `key: null` round-trips through the placeholder string. | -| TOML comments | Lost on roundtrip. JSON has no comments. Acceptable — comments are author intent, not schema data. | - -**Out of scope**: validation (delegated to `jsonschema`), schema authoring helpers, type generation. +| Case | Behavior | +| ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Strings shaped like TOML literals (`"true"`, `"42"`, `"2026-05-04"`, `"inf"`) | Encoder always quotes; decoder parses as strings. No collision. | +| Hand-written unquoted TOML datetime (`default = 2026-05-04`) | Decoder converts `Toml::Datetime` to a JSON string. TOML form is normalized to quoted on re-encode. | +| Hand-written unquoted TOML local time (`default = 09:30:00`) | Same as datetime — converted to JSON string `"09:30:00"`. | +| Empty root schema `{}` | Encodes to empty TOML, decodes back to `{}`. | +| Heterogeneous arrays | TOML 1.1 spec explicitly allows `numbers = [0.1, 0.2, 1, 2]`. The `toml` crate honors this. | +| `serde_json` u64 within i64 range | Roundtrips losslessly via `Toml::Integer`. | +| f64 precision (subnormals, `f64::MIN_POSITIVE`, `f64::MAX`, `0.1 + 0.2`) | Rust `toml` crate uses Ryū-style shortest-roundtrip serialization — values roundtrip exactly. Concern raised in [toml-lang/toml#44](https://github.com/toml-lang/toml/issues/44) does not apply in practice. | +| Unicode in strings (`"café ☕ — 日本語 — 🚀"`) | Roundtrips losslessly. | +| Empty string values (`""`) | Roundtrip. | +| Strings with embedded newlines | Encoded as TOML multi-line basic strings (`"""..."""`); decoded back to JSON strings with `\n`. | +| TOML `+inf`, `-inf`, `nan` floats on decode | **Error.** See "Per-direction policy table" above — `serde_json::Number::from_f64` rejects them, and `jsonschema` never sees them, so erroring early is the only honest behavior. | +| TOML datetime on decode (any of 4 variants) | **JSON string** in canonical RFC 3339 form. JSON Schema uses strings + `format` for dates. | +| Absent TOML key vs. JSON `null` | TOML grammar has no syntax for "key with no value" (parse error). Absent TOML key → absent JSON object key. JSON `key: null` round-trips through the placeholder string. | +| TOML comments | Lost on roundtrip. JSON has no comments. Acceptable — comments are author intent, not schema data. | + +**Out of scope**: validation (delegated to `jsonschema`), schema authoring +helpers, type generation. ### Prototype findings Four `rust-script` prototypes validate the design end-to-end: -- `scripts/test_tomljson.rs` — encoding-rules baseline, 37 cases via a generic JSON↔TOML Value walker. Establishes the canonical TOML form for every JSON Schema construct. -- `scripts/test_tomljson_writer.rs` — JSON→TOML (encode) spike, 18 cases via `toml_writer` dispatcher. Validates the chosen production approach for the encode direction. -- `scripts/test_tomljson_decode.rs` — TOML→JSON (decode) spike, 31 cases. Explicitly exercises the per-direction policy table: all four datetime variants, +inf/-inf/NaN errors, placeholder substitution at scalar / array / nested positions, `__root__` unwrap, custom placeholder via `TomlJsonOptions`, inline tables, arrays-of-tables. -- `scripts/test_path_scoped_validation.rs` — path-scoped validation spike, 12 cases against `jsonschema 0.46` + `globset 0.4`. Validates partition (strip `x-mdvs.allowed`/`x-mdvs.required` cleanly), per-file overlay synthesis, double-validation merging type errors (global) with presence errors (overlay), `[fields].ignore` permissiveness, deep glob matching, and combined-violation reports. -- `scripts/test_violation_mapping.rs` — `ValidationError → ViolationKind` mapping spike, 22 cases. Confirms every `ErrorKind` variant the Wave B translator emits maps cleanly to mdvs's existing `ViolationKind` enum (`MissingRequired`, `WrongType`, `Disallowed`, `NullNotAllowed`, `InvalidCategory`, `OutOfRange`). Surfaces the only non-mechanical case: `Type { ... }` against a `null` instance routes to `NullNotAllowed` instead of `WrongType`. - -The encoding rules below are exercised across: basic types, constraints, `enum` heterogeneity (including `null`), `oneOf`/`anyOf`/`allOf`/`not`, `$defs`/`$ref`, `if`/`then`/`else`, deep nesting, `x-mdvs-*` extensions, boolean schemas, number fidelity, custom null placeholders via `TomlJsonOptions`, placeholder-collision detection, strings shaped like TOML literals, unquoted TOML datetime input, empty root schema, integer-range boundaries (i64::MAX/MIN roundtrip; u64 > i64::MAX errors), and a realistic composite. All cases pass; canonical hand-written TOML for the composite decodes to the same JSON. +- `scripts/test_tomljson.rs` — encoding-rules baseline, 37 cases via a generic + JSON↔TOML Value walker. Establishes the canonical TOML form for every JSON + Schema construct. +- `scripts/test_tomljson_writer.rs` — JSON→TOML (encode) spike, 18 cases via + `toml_writer` dispatcher. Validates the chosen production approach for the + encode direction. +- `scripts/test_tomljson_decode.rs` — TOML→JSON (decode) spike, 31 cases. + Explicitly exercises the per-direction policy table: all four datetime + variants, +inf/-inf/NaN errors, placeholder substitution at scalar / array / + nested positions, `__root__` unwrap, custom placeholder via `TomlJsonOptions`, + inline tables, arrays-of-tables. +- `scripts/test_path_scoped_validation.rs` — path-scoped validation spike, 12 + cases against `jsonschema 0.46` + `globset 0.4`. Validates partition (strip + `x-mdvs.allowed`/`x-mdvs.required` cleanly), per-file overlay synthesis, + double-validation merging type errors (global) with presence errors (overlay), + `[fields].ignore` permissiveness, deep glob matching, and combined-violation + reports. +- `scripts/test_violation_mapping.rs` — `ValidationError → ViolationKind` + mapping spike, 22 cases. Confirms every `ErrorKind` variant the Wave B + translator emits maps cleanly to mdvs's existing `ViolationKind` enum + (`MissingRequired`, `WrongType`, `Disallowed`, `NullNotAllowed`, + `InvalidCategory`, `OutOfRange`). Surfaces the only non-mechanical case: + `Type { ... }` against a `null` instance routes to `NullNotAllowed` instead of + `WrongType`. + +The encoding rules below are exercised across: basic types, constraints, `enum` +heterogeneity (including `null`), `oneOf`/`anyOf`/`allOf`/`not`, `$defs`/`$ref`, +`if`/`then`/`else`, deep nesting, `x-mdvs-*` extensions, boolean schemas, number +fidelity, custom null placeholders via `TomlJsonOptions`, placeholder-collision +detection, strings shaped like TOML literals, unquoted TOML datetime input, +empty root schema, integer-range boundaries (i64::MAX/MIN roundtrip; u64 > +i64::MAX errors), and a realistic composite. All cases pass; canonical +hand-written TOML for the composite decodes to the same JSON. Decisions confirmed: -- **Null encoding** — JSON `null` is represented by a string placeholder (default `"__null__"`, configurable). The placeholder occupies the slot uniformly: `default = "__null__"`, `enum = [1, 2, "__null__"]`, `const = "__null__"`. Arrays stay homogeneous-looking instead of mixing in inline tables. See "Null encoding" section above for the directive form and embedded API. -- **Boolean-schema root** — top-level boolean schemas wrap under `__root__ = true|false`. Dunder convention matches the null sentinel. -- **Heterogeneous arrays** — `enum = [1, "two", true]` works directly; the [TOML 1.1 spec](https://toml.io/en/v1.1.0) explicitly permits mixed-type arrays and the `toml` crate honors this. -- **Number fidelity** — integer vs float distinction preserved through `serde_json::Number` (i64 ↔ `Toml::Integer`, f64 ↔ `Toml::Float`). -- **Integer range** — i64::MAX/MIN roundtrip exactly; values in `(i64::MAX, u64::MAX]` error on encode (TOML spec compliance). -- **String/literal disambiguation** — strings shaped like TOML literals (`"42"`, `"true"`, `"2026-05-04"`) always emitted as quoted strings. Hand-written unquoted TOML datetimes accepted on input (decoded to JSON strings). -- **TOML→JSON datetime policy** — all four TOML datetime variants (`Date`, `Time`, `LocalDateTime`, `OffsetDateTime`) decode to JSON strings using their canonical RFC 3339 representation. JSON Schema represents dates as strings + `format` keyword anyway. -- **TOML→JSON `+inf` / `-inf` / `NaN` policy** — error on decode. `serde_json::Number::from_f64` rejects them; the validation layer can never see them. Erroring early surfaces the limitation at the right place. Storage layers (Parquet, Arrow, LanceDB) support these values; the validation pipeline (jsonschema crate) does not. Users wanting "no upper/lower bound" should omit `maximum`/`minimum` per JSON Schema convention. -- **`nullable: bool` in mdvs is unrelated to TOML's data model** — it controls whether *YAML frontmatter values* may be null, not TOML config values. mdvs does not extend or reinterpret TOML grammar anywhere. -- **Reserved-looking keys** — `$schema`, `$id`, `$ref`, `$defs` auto-quote; `x-mdvs-*` (hyphens allowed in bare keys) does not need quoting. +- **Null encoding** — JSON `null` is represented by a string placeholder + (default `"__null__"`, configurable). The placeholder occupies the slot + uniformly: `default = "__null__"`, `enum = [1, 2, "__null__"]`, + `const = "__null__"`. Arrays stay homogeneous-looking instead of mixing in + inline tables. See "Null encoding" section above for the directive form and + embedded API. +- **Boolean-schema root** — top-level boolean schemas wrap under + `__root__ = true|false`. Dunder convention matches the null sentinel. +- **Heterogeneous arrays** — `enum = [1, "two", true]` works directly; the + [TOML 1.1 spec](https://toml.io/en/v1.1.0) explicitly permits mixed-type + arrays and the `toml` crate honors this. +- **Number fidelity** — integer vs float distinction preserved through + `serde_json::Number` (i64 ↔ `Toml::Integer`, f64 ↔ `Toml::Float`). +- **Integer range** — i64::MAX/MIN roundtrip exactly; values in + `(i64::MAX, u64::MAX]` error on encode (TOML spec compliance). +- **String/literal disambiguation** — strings shaped like TOML literals (`"42"`, + `"true"`, `"2026-05-04"`) always emitted as quoted strings. Hand-written + unquoted TOML datetimes accepted on input (decoded to JSON strings). +- **TOML→JSON datetime policy** — all four TOML datetime variants (`Date`, + `Time`, `LocalDateTime`, `OffsetDateTime`) decode to JSON strings using their + canonical RFC 3339 representation. JSON Schema represents dates as strings + + `format` keyword anyway. +- **TOML→JSON `+inf` / `-inf` / `NaN` policy** — error on decode. + `serde_json::Number::from_f64` rejects them; the validation layer can never + see them. Erroring early surfaces the limitation at the right place. Storage + layers (Parquet, Arrow, LanceDB) support these values; the validation pipeline + (jsonschema crate) does not. Users wanting "no upper/lower bound" should omit + `maximum`/`minimum` per JSON Schema convention. +- **`nullable: bool` in mdvs is unrelated to TOML's data model** — it controls + whether _YAML frontmatter values_ may be null, not TOML config values. mdvs + does not extend or reinterpret TOML grammar anywhere. +- **Reserved-looking keys** — `$schema`, `$id`, `$ref`, `$defs` auto-quote; + `x-mdvs-*` (hyphens allowed in bare keys) does not need quoting. - **`$defs` nesting** — `["$defs".address]` and deeper paths work as expected. Open issues for the crate to handle (not blockers): -- **Array-of-tables sub-key emission** — `toml::to_string` emits `[anyOf.properties.kind]` (singular path under `[[anyOf]]`) which TOML semantics correctly bind to the most recent array element, but is visually confusing in hand-written form. The crate should prefer explicit `[[anyOf]]` re-headers and document the binding rule. -- **Key ordering** — `toml::to_string` sorts alphabetically. Acceptable for a canonical form (deterministic output) but loses input ordering. If we want to preserve author intent on roundtrip, switch to `toml_edit`. -- **Placeholder collision** — when the schema legitimately contains a string equal to the configured null placeholder, or a top-level key named `__root__`, the encoder errors and the caller picks a different placeholder via `TomlJsonOptions::null_placeholder`. -- **Verbose float emission** — the `toml` crate prints small/large floats in long decimal form rather than scientific notation (e.g., `f64::MIN_POSITIVE` renders as a 300+ digit decimal). Roundtrip-correct but unreadable. Mitigation: switch to `toml_edit` or post-process float emission for canonical output. +- **Array-of-tables sub-key emission** — `toml::to_string` emits + `[anyOf.properties.kind]` (singular path under `[[anyOf]]`) which TOML + semantics correctly bind to the most recent array element, but is visually + confusing in hand-written form. The crate should prefer explicit `[[anyOf]]` + re-headers and document the binding rule. +- **Key ordering** — `toml::to_string` sorts alphabetically. Acceptable for a + canonical form (deterministic output) but loses input ordering. If we want to + preserve author intent on roundtrip, switch to `toml_edit`. +- **Placeholder collision** — when the schema legitimately contains a string + equal to the configured null placeholder, or a top-level key named `__root__`, + the encoder errors and the caller picks a different placeholder via + `TomlJsonOptions::null_placeholder`. +- **Verbose float emission** — the `toml` crate prints small/large floats in + long decimal form rather than scientific notation (e.g., `f64::MIN_POSITIVE` + renders as a 300+ digit decimal). Roundtrip-correct but unreadable. + Mitigation: switch to `toml_edit` or post-process float emission for canonical + output. ## Workspace restructure (Wave A) @@ -1260,7 +1637,8 @@ mdvs/ └── ... ``` -Existing paths in `src/` move under `crates/mdvs/src/`. The root `Cargo.toml` becomes: +Existing paths in `src/` move under `crates/mdvs/src/`. The root `Cargo.toml` +becomes: ```toml [workspace] @@ -1268,7 +1646,9 @@ resolver = "3" members = ["crates/mdvs", "crates/tomljson"] ``` -The current `[package]` block (with `include = ["src/", "skills/", ...]` and all dependencies) moves to `crates/mdvs/Cargo.toml`. Paths in `include` stay relative to that crate's directory. +The current `[package]` block (with `include = ["src/", "skills/", ...]` and all +dependencies) moves to `crates/mdvs/Cargo.toml`. Paths in `include` stay +relative to that crate's directory. mdvs depends on tomljson via path-with-version dep: @@ -1277,21 +1657,24 @@ mdvs depends on tomljson via path-with-version dep: tomljson = { path = "../tomljson", version = "0.1.0" } ``` -`cargo publish` works for both crates separately (publish `tomljson` first since `mdvs` depends on it). +`cargo publish` works for both crates separately (publish `tomljson` first since +`mdvs` depends on it). ### CI/CD changes -| File | Change | -|---|---| -| `.github/workflows/ci.yml` | **No change.** `cargo build/test/clippy/fmt`, `cargo audit`, `cargo deny check` already operate at the workspace root; new crates are picked up transparently. | -| `.github/workflows/bump.yml` | Update the publish step from `cargo publish --no-verify` to publish both crates in dependency order: `cargo publish -p tomljson --no-verify && cargo publish -p mdvs --no-verify`. (Initially, `tomljson` may not be published — only `mdvs` ships. Add `tomljson` once it stabilizes.) | -| `.github/workflows/release.yml` | **Regenerate via `dist init`.** This file is auto-generated by `cargo-dist`. After the workspace move, re-run `dist init` so cargo-dist correctly identifies which workspace member ships binaries (`mdvs`). cargo-dist supports workspaces natively. | -| `cog.toml` | Update the `pre_bump_hooks` sed script. Currently rewrites the root `Cargo.toml`'s version: `sed -i 's/^version = ".*"/version = "{{version}}"/' Cargo.toml`. Becomes per-crate (only `mdvs`'s version is bumped during a release; `tomljson` follows its own cadence): `sed -i 's/^version = ".*"/version = "{{version}}"/' crates/mdvs/Cargo.toml`. | -| `.github/workflows/book.yml` | No change — operates on `book/` directory, unaffected. | -| `.github/workflows/commits.yml` | No change — operates on commit messages. | -| `.github/workflows/claude*.yml` | No change — Claude integration, repo-structure agnostic. | +| File | Change | +| ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `.github/workflows/ci.yml` | **No change.** `cargo build/test/clippy/fmt`, `cargo audit`, `cargo deny check` already operate at the workspace root; new crates are picked up transparently. | +| `.github/workflows/bump.yml` | Update the publish step from `cargo publish --no-verify` to publish both crates in dependency order: `cargo publish -p tomljson --no-verify && cargo publish -p mdvs --no-verify`. (Initially, `tomljson` may not be published — only `mdvs` ships. Add `tomljson` once it stabilizes.) | +| `.github/workflows/release.yml` | **Regenerate via `dist init`.** This file is auto-generated by `cargo-dist`. After the workspace move, re-run `dist init` so cargo-dist correctly identifies which workspace member ships binaries (`mdvs`). cargo-dist supports workspaces natively. | +| `cog.toml` | Update the `pre_bump_hooks` sed script. Currently rewrites the root `Cargo.toml`'s version: `sed -i 's/^version = ".*"/version = "{{version}}"/' Cargo.toml`. Becomes per-crate (only `mdvs`'s version is bumped during a release; `tomljson` follows its own cadence): `sed -i 's/^version = ".*"/version = "{{version}}"/' crates/mdvs/Cargo.toml`. | +| `.github/workflows/book.yml` | No change — operates on `book/` directory, unaffected. | +| `.github/workflows/commits.yml` | No change — operates on commit messages. | +| `.github/workflows/claude*.yml` | No change — Claude integration, repo-structure agnostic. | -**Tag/version semantics**: `v` tags continue to refer to `mdvs`'s version. `tomljson` gets its own tags if/when it's published independently (e.g., `tomljson-v0.1.0`). +**Tag/version semantics**: `v` tags continue to refer to `mdvs`'s +version. `tomljson` gets its own tags if/when it's published independently +(e.g., `tomljson-v0.1.0`). **Lockfile**: `Cargo.lock` stays at the workspace root and covers both crates. @@ -1300,65 +1683,122 @@ tomljson = { path = "../tomljson", version = "0.1.0" } ### Wave A — `tomljson` crate 1. Restructure repo as Cargo workspace; move existing crate to `crates/mdvs/`. -2. Update CI/CD per the table above (`bump.yml` publish step, `release.yml` regenerate via `dist init`, `cog.toml` sed path). -3. Create `crates/tomljson/` skeleton (`Cargo.toml` with deps on `toml`, `toml_writer`, `serde_json`; `lib.rs`). +2. Update CI/CD per the table above (`bump.yml` publish step, `release.yml` + regenerate via `dist init`, `cog.toml` sed path). +3. Create `crates/tomljson/` skeleton (`Cargo.toml` with deps on `toml`, + `toml_writer`, `serde_json`; `lib.rs`). 4. Implement the serialize dispatcher (`src/ser.rs`): - - Pre-flight `assert_encodable` walk: rejects `u64 > i64::MAX` and strings equal to the placeholder. - - Inline emitter for scalars and inline arrays/tables (used inside `enum`, `oneOf`, etc.). - - Document-level emitter that separates inline keys from sub-tables, emits proper `[section]` and `[[array.of.tables]]` headers. + - Pre-flight `assert_encodable` walk: rejects `u64 > i64::MAX` and strings + equal to the placeholder. + - Inline emitter for scalars and inline arrays/tables (used inside `enum`, + `oneOf`, etc.). + - Document-level emitter that separates inline keys from sub-tables, emits + proper `[section]` and `[[array.of.tables]]` headers. - Top-level wrapper for non-table roots → `__root__ = `. - - Optimization: skip emitting empty parent table headers (when a section has no direct keys, only sub-sections). + - Optimization: skip emitting empty parent table headers (when a section has + no direct keys, only sub-sections). 5. Implement the deserialize tree walker (`src/de.rs`): - `toml::from_str` → `toml::Value`. - Walk the tree substituting placeholder strings → `Json::Null`. - Detect root `__root__`-wrapped tables → unwrap. - `serde_json::from_value` to user's target type. 6. Lift the third prototype's test cases into `crates/tomljson/tests/`: - - `test_tomljson.rs` (37 cases, encoding-rules baseline). Steps 4 and 5 already lifted `test_tomljson_writer.rs` (18 cases) and `test_tomljson_decode.rs` (31 cases). -7. Write the `README.md` covering scope, API surface, encoding rules, and motivating use case (JSON Schema in TOML). -8. Optionally publish to crates.io once the API is stable (independent of mdvs's cadence). + - `test_tomljson.rs` (37 cases, encoding-rules baseline). Steps 4 and 5 + already lifted `test_tomljson_writer.rs` (18 cases) and + `test_tomljson_decode.rs` (31 cases). +7. Write the `README.md` covering scope, API surface, encoding rules, and + motivating use case (JSON Schema in TOML). +8. Optionally publish to crates.io once the API is stable (independent of mdvs's + cadence). #### Standalone API / `$tomljson-null` directive — dropped -An earlier design proposed a self-describing standalone API: a TOML document would carry its own placeholder via a top-level `"$tomljson-null" = ""` directive (or, alternatively, a magic comment header), so receiving tools could decode without prior knowledge of the placeholder. +An earlier design proposed a self-describing standalone API: a TOML document +would carry its own placeholder via a top-level +`"$tomljson-null" = ""` directive (or, alternatively, a magic +comment header), so receiving tools could decode without prior knowledge of the +placeholder. **Both options were rejected:** -- An in-band directive key (`$tomljson-null` at root) collides with arbitrary JSON data — non–JSON-Schema documents can legitimately have a top-level key with that name. Encoder would emit duplicate keys; decoder would silently strip user data. -- A magic comment header (e.g. `#:tomljson:null = "..."` as the first comment) invents a non-standard convention with no precedent in TOML, and depends on raw-string preprocessing that's fragile to formatters. -For v0.1, `tomljson` exposes only the explicit `*_with_options` API. Users coordinate the placeholder out-of-band (in their own docs, sibling configs, or a project-wide convention). If a concrete need for self-describing TOML emerges later — informed by real consumers, not pre-emptive design — we add it then with whatever convention proves necessary. +- An in-band directive key (`$tomljson-null` at root) collides with arbitrary + JSON data — non–JSON-Schema documents can legitimately have a top-level key + with that name. Encoder would emit duplicate keys; decoder would silently + strip user data. +- A magic comment header (e.g. `#:tomljson:null = "..."` as the first comment) + invents a non-standard convention with no precedent in TOML, and depends on + raw-string preprocessing that's fragile to formatters. + +For v0.1, `tomljson` exposes only the explicit `*_with_options` API. Users +coordinate the placeholder out-of-band (in their own docs, sibling configs, or a +project-wide convention). If a concrete need for self-describing TOML emerges +later — informed by real consumers, not pre-emptive design — we add it then with +whatever convention proves necessary. ### Wave B — mdvs adoption of JSON Schema 1. Add `jsonschema` dependency. Spike error quality on representative cases. -2. New module `src/schema/json_schema.rs` — translator from `mdvs.toml` DSL to canonical JSON Schema, plus `validate_mdvs_schema` gate (allow-list + structural rules) used by both `init --schema` and `check --schema`. Uses `tomljson` for any TOML I/O of schema documents (export `--format toml`, loading a `.toml` schema via `--schema`). -3. New module `src/preprocess.rs` — preprocessor pipeline (`coerce_to_string`, `widen_int_to_float`). -4. Replace `ConstraintKind::validate_value` and `type_matches` with `jsonschema::Validator::validate`. +2. New module `src/schema/json_schema.rs` — translator from `mdvs.toml` DSL to + canonical JSON Schema, plus `validate_mdvs_schema` gate (allow-list + + structural rules) used by both `init --schema` and `check --schema`. Uses + `tomljson` for any TOML I/O of schema documents (export `--format toml`, + loading a `.toml` schema via `--schema`). +3. New module `src/preprocess.rs` — preprocessor pipeline (`coerce_to_string`, + `widen_int_to_float`). +4. Replace `ConstraintKind::validate_value` and `type_matches` with + `jsonschema::Validator::validate`. 5. Map `jsonschema` errors to `ViolationKind`. See "Error mapping" section. -6. Remove hand-rolled validation in `categories.rs`, `range.rs` (keep `validate_for_type`, inference). -7. Add `min_length`, `max_length`, `pattern` to `Constraints` serde + translator + inference. -8. **Fix the YAML→JSON conversion silent-drop**: in `src/discover/scan.rs:126`, replace `Pod::deserialize().ok()?` with explicit error propagation. Add new `ViolationKind::FrontmatterUnrepresentable { field, reason }` for values that can't fit in `serde_json::Value` (NaN/inf, non-string keys). See "YAML→JSON conversion" section above. -9. New `mdvs export-schema` command (uses `tomljson::to_string` for canonical TOML output, `serde_json::to_string_pretty` for JSON output). +6. Remove hand-rolled validation in `categories.rs`, `range.rs` (keep + `validate_for_type`, inference). +7. Add `min_length`, `max_length`, `pattern` to `Constraints` serde + + translator + inference. +8. **Fix the YAML→JSON conversion silent-drop**: in `src/discover/scan.rs:126`, + replace `Pod::deserialize().ok()?` with explicit error propagation. Add new + `ViolationKind::FrontmatterUnrepresentable { field, reason }` for values that + can't fit in `serde_json::Value` (NaN/inf, non-string keys). See "YAML→JSON + conversion" section above. +9. New `mdvs export-schema` command (uses `tomljson::to_string` for canonical + TOML output, `serde_json::to_string_pretty` for JSON output). 10. Add `--schema PATH` flag on **two commands only**: - - `init`: one-shot import. Translates schema → DSL via `canonical_to_dsl`, writes inline `[[fields.field]]`. - - `check`: read-only override. Schema is the sole source of fields/preprocess for that invocation. `mdvs.toml` may be absent (defaults applied). - No flag on `update`, `build`, `search`, `info`, `clean`. -11. Implement file-extension-sniffing schema loader (`.json` → `serde_json::from_str`; `.toml` → `tomljson::from_str`). No `schema_ref` field on `MdvsToml`. -12. Implement schema resolution as the binary helper documented in "Schema sourcing": `cli_override.or_else(|| dsl_to_canonical(mdvs_toml))`. -13. Implement per-file overlay synthesis (path-scoped validation) and double-validation. See "Path-scoped validation" section. -14. Update build metadata to record the resolved schema's content hash. Trigger `--force` requirement when the hash changes between builds (same mechanism as model/chunking change detection). + - `init`: one-shot import. Translates schema → DSL via `canonical_to_dsl`, + writes inline `[[fields.field]]`. + - `check`: read-only override. Schema is the sole source of + fields/preprocess for that invocation. `mdvs.toml` may be absent (defaults + applied). No flag on `update`, `build`, `search`, `info`, `clean`. +11. Implement file-extension-sniffing schema loader (`.json` → + `serde_json::from_str`; `.toml` → `tomljson::from_str`). No `schema_ref` + field on `MdvsToml`. +12. Implement schema resolution as the binary helper documented in "Schema + sourcing": `cli_override.or_else(|| dsl_to_canonical(mdvs_toml))`. +13. Implement per-file overlay synthesis (path-scoped validation) and + double-validation. See "Path-scoped validation" section. +14. Update build metadata to record the resolved schema's content hash. Trigger + `--force` requirement when the hash changes between builds (same mechanism + as model/chunking change detection). ### Wave C — type naming + object flattening -1. Rename `FieldType` variants and `mdvs.toml` type strings to lowercase JSON Schema names. -2. Implement object flattening per TODO-0097 (`[[fields.field]]` with dotted `name`). - **Translator update required.** Wave B left the Object arm of `type_subschema` (in `crates/mdvs/src/schema/json_schema.rs`) deliberately loose — emits `{"type": "object", "additionalProperties": true}` and does not project children. This was a knowing placeholder to preserve the existing `type_matches(Object(_), Value::Object(_))` semantics until flattening lands. After Wave C, `FieldType::Object` is gone (children become top-level dotted-name scalar fields), so the translator's Object arm becomes unreachable / removable. Verify children are validated end-to-end after flattening. -3. Migration path for existing `mdvs.toml` files (one-shot `mdvs migrate-config`?). +1. Rename `FieldType` variants and `mdvs.toml` type strings to lowercase JSON + Schema names. +2. Implement object flattening per TODO-0097 (`[[fields.field]]` with dotted + `name`). **Translator update required.** Wave B left the Object arm of + `type_subschema` (in `crates/mdvs/src/schema/json_schema.rs`) deliberately + loose — emits `{"type": "object", "additionalProperties": true}` and does not + project children. This was a knowing placeholder to preserve the existing + `type_matches(Object(_), Value::Object(_))` semantics until flattening lands. + After Wave C, `FieldType::Object` is gone (children become top-level + dotted-name scalar fields), so the translator's Object arm becomes + unreachable / removable. Verify children are validated end-to-end after + flattening. +3. Migration path for existing `mdvs.toml` files (one-shot + `mdvs migrate-config`?). 4. Update all docs, examples, `example_kb`. ### Wave B — execution plan (sequencing) -The 14-step list above is correct but not in dependency order. This appendix gives a per-step view: prerequisites, files touched, test additions, definition of done. References the same step numbers — no renumbering. +The 14-step list above is correct but not in dependency order. This appendix +gives a per-step view: prerequisites, files touched, test additions, definition +of done. References the same step numbers — no renumbering. #### Dependency graph @@ -1372,260 +1812,487 @@ The 14-step list above is correct but not in dependency order. This appendix giv 12 → 11 → 10 → 9 → 13 → 14 (CLI surface + per-file synthesis + metadata) ``` -**Rationale.** Steps 1–2 are foundation — the `jsonschema` dependency and the DSL→canonical translator must exist before anything else can call into them. Step 5 (error mapping) does **not** strictly block step 4 (engine swap): step 4 can land with raw `ValidationError` output temporarily and step 5 layers the `ViolationKind` mapping on top. Step 8 (YAML→JSON silent-drop fix) is independent of the validation swap but slots in before the CLI surface because it adds a new `ViolationKind::FrontmatterUnrepresentable` that step 5's mapping table must include. Steps 9–14 form the user-facing surface and depend on the engine being swapped. +**Rationale.** Steps 1–2 are foundation — the `jsonschema` dependency and the +DSL→canonical translator must exist before anything else can call into them. +Step 5 (error mapping) does **not** strictly block step 4 (engine swap): step 4 +can land with raw `ValidationError` output temporarily and step 5 layers the +`ViolationKind` mapping on top. Step 8 (YAML→JSON silent-drop fix) is +independent of the validation swap but slots in before the CLI surface because +it adds a new `ViolationKind::FrontmatterUnrepresentable` that step 5's mapping +table must include. Steps 9–14 form the user-facing surface and depend on the +engine being swapped. #### Per-step blocks ##### Step 1 — Add `jsonschema` dependency, spike error quality -**Prereqs:** none. -**Files touched:** +**Prereqs:** none. **Files touched:** + - `crates/mdvs/Cargo.toml` — add `jsonschema = "0.x"` (latest stable). -- `crates/mdvs/Cargo.toml` — add `tomljson = { path = "../tomljson" }` (needed by step 2; cheaper to add now). +- `crates/mdvs/Cargo.toml` — add `tomljson = { path = "../tomljson" }` (needed + by step 2; cheaper to add now). **Test additions:** -- New `scripts/test_jsonschema_errors.rs` rust-script — battery of representative violation cases (wrong type, missing required, out-of-range, enum miss, length, pattern), prints the `ValidationError` shape verbatim to stderr for human inspection. Sole purpose: confirm the error shapes carry enough info to drive step 5's mapping. **Not** committed as a permanent test — this is a design spike. -**Definition of done:** crate compiles with `jsonschema` linked; spike script runs and prints all error shapes for review. +- New `scripts/test_jsonschema_errors.rs` rust-script — battery of + representative violation cases (wrong type, missing required, out-of-range, + enum miss, length, pattern), prints the `ValidationError` shape verbatim to + stderr for human inspection. Sole purpose: confirm the error shapes carry + enough info to drive step 5's mapping. **Not** committed as a permanent test — + this is a design spike. + +**Definition of done:** crate compiles with `jsonschema` linked; spike script +runs and prints all error shapes for review. ##### Step 2 — DSL → canonical JSON Schema translator + validation gate -**Prereqs:** step 1. -**Files touched:** -- `crates/mdvs/src/schema/json_schema.rs` (new) — `dsl_to_canonical(&MdvsToml) -> serde_json::Value` translator and `validate_mdvs_schema(&serde_json::Value) -> Result<()>` gate. `MDVS_KEYS` const lives here. +**Prereqs:** step 1. **Files touched:** + +- `crates/mdvs/src/schema/json_schema.rs` (new) — + `dsl_to_canonical(&MdvsToml) -> serde_json::Value` translator and + `validate_mdvs_schema(&serde_json::Value) -> Result<()>` gate. `MDVS_KEYS` + const lives here. - `crates/mdvs/src/schema/mod.rs` — declare new module. **Test additions:** -- Inline `#[cfg(test)] mod tests` in `json_schema.rs`: round-trip every constraint kind currently in `Constraints` (categories, range — length and pattern come in step 7), assert canonical JSON shape matches expectation. -- Validation-gate tests: hand-written schemas hitting each rejected keyword (`oneOf`, `$ref`, `if/then/else`, `patternProperties`, etc.) — confirm each rejection. -**Definition of done:** translator produces canonical JSON Schema for every shape currently expressible in `mdvs.toml`; gate accepts the allow-list and rejects the deny-list; `cargo clippy --all-targets` clean. +- Inline `#[cfg(test)] mod tests` in `json_schema.rs`: round-trip every + constraint kind currently in `Constraints` (categories, range — length and + pattern come in step 7), assert canonical JSON shape matches expectation. +- Validation-gate tests: hand-written schemas hitting each rejected keyword + (`oneOf`, `$ref`, `if/then/else`, `patternProperties`, etc.) — confirm each + rejection. + +**Definition of done:** translator produces canonical JSON Schema for every +shape currently expressible in `mdvs.toml`; gate accepts the allow-list and +rejects the deny-list; `cargo clippy --all-targets` clean. ##### Step 4 — Replace `validate_value` and `type_matches` with `jsonschema::Validator` -**Prereqs:** steps 1, 2. -**Files touched:** -- `crates/mdvs/src/cmd/check.rs:245–249` — `validate()` builds a `jsonschema::Validator` from `dsl_to_canonical(config)` once, then calls `.validate(value)` per file/field. -- `crates/mdvs/src/cmd/check.rs:339, :456` — remove the `type_matches` call site and its definition. -- `crates/mdvs/src/schema/constraints/mod.rs:154–165` — `ConstraintKind::validate_value` becomes a thin wrapper that delegates to the new validator (kept for inference paths in step 6). +**Prereqs:** steps 1, 2. **Files touched:** + +- `crates/mdvs/src/cmd/check.rs:245–249` — `validate()` builds a + `jsonschema::Validator` from `dsl_to_canonical(config)` once, then calls + `.validate(value)` per file/field. +- `crates/mdvs/src/cmd/check.rs:339, :456` — remove the `type_matches` call site + and its definition. +- `crates/mdvs/src/schema/constraints/mod.rs:154–165` — + `ConstraintKind::validate_value` becomes a thin wrapper that delegates to the + new validator (kept for inference paths in step 6). **Test additions:** -- Move existing `#[cfg(test)] mod tests` cases from `check.rs:1082+` and `constraints/mod.rs:201–340` to assert against the new code path. Same input → same `ViolationKind` output (mapping is step 5, but the temporary path can stringify `ValidationError`). -**Definition of done:** `cargo test` passes against the new engine; all existing inline tests reading `ConstraintKind::validate_value` keep passing. +- Move existing `#[cfg(test)] mod tests` cases from `check.rs:1082+` and + `constraints/mod.rs:201–340` to assert against the new code path. Same input → + same `ViolationKind` output (mapping is step 5, but the temporary path can + stringify `ValidationError`). + +**Definition of done:** `cargo test` passes against the new engine; all existing +inline tests reading `ConstraintKind::validate_value` keep passing. ##### Step 5 — Map `jsonschema::ValidationError` → `ViolationKind` -**Prereqs:** step 4. (Also implicitly depends on step 8 — its new `FrontmatterUnrepresentable` variant must be in the mapping table.) -**Files touched:** -- `crates/mdvs/src/cmd/check.rs` — add `map_validation_error(&ValidationError) -> ViolationKind` helper; replace the temporary `ValidationError → string` path from step 4 with this. Path-from-pointer logic also lives here (`ValidationError::instance_path` → field name). -- `crates/mdvs/src/output.rs:172–186` — extend `ViolationKind` if the spike from step 1 surfaces a case the current variants can't express (TBD: see open question 5 below). +**Prereqs:** step 4. (Also implicitly depends on step 8 — its new +`FrontmatterUnrepresentable` variant must be in the mapping table.) **Files +touched:** + +- `crates/mdvs/src/cmd/check.rs` — add + `map_validation_error(&ValidationError) -> ViolationKind` helper; replace the + temporary `ValidationError → string` path from step 4 with this. + Path-from-pointer logic also lives here (`ValidationError::instance_path` → + field name). +- `crates/mdvs/src/output.rs:172–186` — extend `ViolationKind` if the spike from + step 1 surfaces a case the current variants can't express (TBD: see open + question 5 below). **Test additions:** -- Lift `scripts/test_violation_mapping.rs` (22 cases) into a permanent test. Location TBD — see open question 5 below. -**Definition of done:** every `ValidationError` shape produced by the canonical schemas in step 2 maps to a `ViolationKind`; mapping table tests pass. +- Lift `scripts/test_violation_mapping.rs` (22 cases) into a permanent test. + Location TBD — see open question 5 below. + +**Definition of done:** every `ValidationError` shape produced by the canonical +schemas in step 2 maps to a `ViolationKind`; mapping table tests pass. ##### Step 6 — Remove hand-rolled validation in `categories.rs`, `range.rs` **Prereqs:** step 5 (so the new path is verified before we delete the old one). **Files touched:** -- `crates/mdvs/src/schema/constraints/categories.rs` — remove the `validate_value` function. Keep `validate_for_type` and inference helpers. -- `crates/mdvs/src/schema/constraints/range.rs` — same: remove `validate_value`, keep inference. -- `crates/mdvs/src/schema/constraints/mod.rs:154–165` — `ConstraintKind::validate_value` deleted (callers from step 4 already stopped using it). -**Test additions:** none — purely deletion. The existing inline tests for these functions are removed alongside their subjects. +- `crates/mdvs/src/schema/constraints/categories.rs` — remove the + `validate_value` function. Keep `validate_for_type` and inference helpers. +- `crates/mdvs/src/schema/constraints/range.rs` — same: remove `validate_value`, + keep inference. +- `crates/mdvs/src/schema/constraints/mod.rs:154–165` — + `ConstraintKind::validate_value` deleted (callers from step 4 already stopped + using it). -**Definition of done:** `cargo test` and `cargo clippy --all-targets` clean; grep for `validate_value` in `schema/constraints/` returns zero hits. +**Test additions:** none — purely deletion. The existing inline tests for these +functions are removed alongside their subjects. + +**Definition of done:** `cargo test` and `cargo clippy --all-targets` clean; +grep for `validate_value` in `schema/constraints/` returns zero hits. ##### Step 3 — Preprocessor pipeline runner + built-in stages -**Prereqs:** step 2 (the pipeline runs **before** validation, but the validation engine doesn't need to call into it for step 3 itself — they're independent modules until step 4 wires them). -**Files touched:** -- `crates/mdvs/src/preprocess.rs` (new) — `Stage1`, `Stage2`, `Stage3` enums; `Pipeline` struct; `coerce_to_string` and `widen_int_to_float` built-ins. Match-based dispatch as documented in the design section. +**Prereqs:** step 2 (the pipeline runs **before** validation, but the validation +engine doesn't need to call into it for step 3 itself — they're independent +modules until step 4 wires them). **Files touched:** + +- `crates/mdvs/src/preprocess.rs` (new) — `Stage1`, `Stage2`, `Stage3` enums; + `Pipeline` struct; `coerce_to_string` and `widen_int_to_float` built-ins. + Match-based dispatch as documented in the design section. - `crates/mdvs/src/lib.rs` — declare module. -- `crates/mdvs/src/cmd/check.rs:245–249` — call the pipeline on the value before handing it to the validator. +- `crates/mdvs/src/cmd/check.rs:245–249` — call the pipeline on the value before + handing it to the validator. **Test additions:** -- Inline `#[cfg(test)] mod tests` in `preprocess.rs`: each built-in's behaviour, plus pipeline composition (stage 1 → stage 2 → stage 3 ordering, no-op when no preprocessors configured). -**Definition of done:** check command runs preprocessors before validation; existing test fixtures still pass; the pipeline can be a no-op (zero stages configured) without overhead. +- Inline `#[cfg(test)] mod tests` in `preprocess.rs`: each built-in's behaviour, + plus pipeline composition (stage 1 → stage 2 → stage 3 ordering, no-op when no + preprocessors configured). + +**Definition of done:** check command runs preprocessors before validation; +existing test fixtures still pass; the pipeline can be a no-op (zero stages +configured) without overhead. ##### Step 7 — Add `min_length`, `max_length`, `pattern` to `Constraints` -**Prereqs:** steps 2, 3. -**Files touched:** -- `crates/mdvs/src/schema/constraints/mod.rs:22–40` — extend `Constraints` struct with `min_length: Option`, `max_length: Option`, `pattern: Option`. Use `#[serde(default, skip_serializing_if = "Option::is_none")]` so they only appear in toml when set. -- `crates/mdvs/src/schema/constraints/mod.rs:48–59` — extend `ConstraintKind` enum with new variants (or fold into existing — TBD by author at the time). -- `crates/mdvs/src/schema/json_schema.rs` — translator emits `minLength`, `maxLength`, `pattern` keywords. -- `crates/mdvs/src/discover/constraints/` (or wherever inference lives) — observe length/pattern during inference. **TBD:** spec line 333 says `discover/constraints/` but exploration found no such directory — author confirms location at step start. +**Prereqs:** steps 2, 3. **Files touched:** + +- `crates/mdvs/src/schema/constraints/mod.rs:22–40` — extend `Constraints` + struct with `min_length: Option`, `max_length: Option`, + `pattern: Option`. Use + `#[serde(default, skip_serializing_if = "Option::is_none")]` so they only + appear in toml when set. +- `crates/mdvs/src/schema/constraints/mod.rs:48–59` — extend `ConstraintKind` + enum with new variants (or fold into existing — TBD by author at the time). +- `crates/mdvs/src/schema/json_schema.rs` — translator emits `minLength`, + `maxLength`, `pattern` keywords. +- `crates/mdvs/src/discover/constraints/` (or wherever inference lives) — + observe length/pattern during inference. **TBD:** spec line 333 says + `discover/constraints/` but exploration found no such directory — author + confirms location at step start. **Test additions:** -- Inline tests in `json_schema.rs`: round-trip a `Constraints` with each new field set. -- Inline tests for inference: scan a fixture directory with strings of varying lengths and a fixed-format string, confirm `min_length`/`max_length`/`pattern` get inferred (or explicitly skipped if inference policy says so). -**Definition of done:** new fields round-trip through `mdvs.toml`; check command rejects strings violating each new constraint with the right `ViolationKind`. +- Inline tests in `json_schema.rs`: round-trip a `Constraints` with each new + field set. +- Inline tests for inference: scan a fixture directory with strings of varying + lengths and a fixed-format string, confirm `min_length`/`max_length`/`pattern` + get inferred (or explicitly skipped if inference policy says so). + +**Definition of done:** new fields round-trip through `mdvs.toml`; check command +rejects strings violating each new constraint with the right `ViolationKind`. ##### Step 8 — Fix YAML→JSON silent-drop, add `FrontmatterUnrepresentable` -**Prereqs:** none (independent of validation swap, but should land **before** step 5's mapping table is finalized). -**Files touched:** -- `crates/mdvs/src/discover/scan.rs:126` — replace `let json: Value = d.deserialize().ok()?;` with explicit error propagation that surfaces the failure as a per-file `FrontmatterUnrepresentable` violation. Decide whether the file is included with the violation or excluded from scan — **TBD at step start.** -- `crates/mdvs/src/output.rs:172–186` — add `ViolationKind::FrontmatterUnrepresentable { field: String, reason: String }` variant. -- `crates/mdvs/src/cmd/check.rs` — render the new variant in the violation output. +**Prereqs:** none (independent of validation swap, but should land **before** +step 5's mapping table is finalized). **Files touched:** + +- `crates/mdvs/src/discover/scan.rs:126` — replace + `let json: Value = d.deserialize().ok()?;` with explicit error propagation + that surfaces the failure as a per-file `FrontmatterUnrepresentable` + violation. Decide whether the file is included with the violation or excluded + from scan — **TBD at step start.** +- `crates/mdvs/src/output.rs:172–186` — add + `ViolationKind::FrontmatterUnrepresentable { field: String, reason: String }` + variant. +- `crates/mdvs/src/cmd/check.rs` — render the new variant in the violation + output. **Test additions:** -- Inline tests in `scan.rs`: fixtures with frontmatter containing NaN, +inf, non-string keys → confirm each surfaces as `FrontmatterUnrepresentable` rather than silently dropping the file. -**Definition of done:** no frontmatter is ever silently dropped; every unrepresentable value surfaces as a violation that step 5 maps cleanly. +- Inline tests in `scan.rs`: fixtures with frontmatter containing NaN, +inf, + non-string keys → confirm each surfaces as `FrontmatterUnrepresentable` rather + than silently dropping the file. + +**Definition of done:** no frontmatter is ever silently dropped; every +unrepresentable value surfaces as a violation that step 5 maps cleanly. ##### Step 12 — Schema resolution helper -**Prereqs:** step 2 (uses `dsl_to_canonical`). -**Files touched:** -- `crates/mdvs/src/schema/json_schema.rs` (or new `crates/mdvs/src/schema/resolve.rs`) — `resolve_schema(cli_override: Option<&Path>, toml: Option<&MdvsToml>) -> Result` implementing `cli_override.or_else(|| dsl_to_canonical(toml))`. +**Prereqs:** step 2 (uses `dsl_to_canonical`). **Files touched:** + +- `crates/mdvs/src/schema/json_schema.rs` (or new + `crates/mdvs/src/schema/resolve.rs`) — + `resolve_schema(cli_override: Option<&Path>, toml: Option<&MdvsToml>) -> Result` + implementing `cli_override.or_else(|| dsl_to_canonical(toml))`. **Test additions:** -- Inline tests: cli override wins; falls back to toml; both absent → defined error. -**Definition of done:** single helper that every schema-consuming code path can call. +- Inline tests: cli override wins; falls back to toml; both absent → defined + error. + +**Definition of done:** single helper that every schema-consuming code path can +call. ##### Step 11 — File-extension-sniffing schema loader -**Prereqs:** step 2. -**Files touched:** -- `crates/mdvs/src/schema/load.rs` (new) — `load_schema(path: &Path) -> Result`. `.json` → `serde_json::from_str`; `.toml` → `tomljson::from_str`. Other extensions → error. +**Prereqs:** step 2. **Files touched:** + +- `crates/mdvs/src/schema/load.rs` (new) — + `load_schema(path: &Path) -> Result`. `.json` → + `serde_json::from_str`; `.toml` → `tomljson::from_str`. Other extensions → + error. **Test additions:** -- Inline tests against fixture files: `.json`, `.toml`, `.yaml` (rejected), unknown extension (rejected). -**Definition of done:** loader handles both formats; rejects unsupported extensions with a clear error. +- Inline tests against fixture files: `.json`, `.toml`, `.yaml` (rejected), + unknown extension (rejected). + +**Definition of done:** loader handles both formats; rejects unsupported +extensions with a clear error. ##### Step 10 — `--schema PATH` on `init` and `check` -**Prereqs:** steps 2, 11, 12. -**Files touched:** -- `crates/mdvs/src/schema/json_schema.rs` — add `canonical_to_dsl(&serde_json::Value) -> Result>` (the reverse of step 2's `dsl_to_canonical`; deferred from step 2 because only `init --schema` consumes it). Lossy where canonical schemas use keywords the DSL can't represent — error rather than silently drop. -- `crates/mdvs/src/cmd/init.rs` — add `--schema PATH` arg. Behaviour: load via step 11, validate via step 2's gate, translate canonical→DSL via `canonical_to_dsl`, write inline `[[fields.field]]` blocks to `mdvs.toml`. Schema is **consumed**, not stored. -- `crates/mdvs/src/cmd/check.rs` — add `--schema PATH` arg. Behaviour: load via step 11, validate via step 2's gate, use as sole source of fields/preprocess for the invocation. Allow `mdvs.toml` to be absent (defaults applied — see open question 4). +**Prereqs:** steps 2, 11, 12. **Files touched:** + +- `crates/mdvs/src/schema/json_schema.rs` — add + `canonical_to_dsl(&serde_json::Value) -> Result>` (the reverse + of step 2's `dsl_to_canonical`; deferred from step 2 because only + `init --schema` consumes it). Lossy where canonical schemas use keywords the + DSL can't represent — error rather than silently drop. +- `crates/mdvs/src/cmd/init.rs` — add `--schema PATH` arg. Behaviour: load via + step 11, validate via step 2's gate, translate canonical→DSL via + `canonical_to_dsl`, write inline `[[fields.field]]` blocks to `mdvs.toml`. + Schema is **consumed**, not stored. +- `crates/mdvs/src/cmd/check.rs` — add `--schema PATH` arg. Behaviour: load via + step 11, validate via step 2's gate, use as sole source of fields/preprocess + for the invocation. Allow `mdvs.toml` to be absent (defaults applied — see + open question 4). **Test additions:** -- Inline tests in `json_schema.rs` for `canonical_to_dsl`: round-trip every `dsl_to_canonical` output back through `canonical_to_dsl` and assert equality with the original `Vec`. Reject test: schemas containing `oneOf` etc. (already rejected by the gate, but `canonical_to_dsl` should defensively error too). -- Inline tests in each command: `--schema` happy path, schema-validation failure, conflicting `mdvs.toml`+`--schema` on `init` (decide policy: error or override). + +- Inline tests in `json_schema.rs` for `canonical_to_dsl`: round-trip every + `dsl_to_canonical` output back through `canonical_to_dsl` and assert equality + with the original `Vec`. Reject test: schemas containing `oneOf` + etc. (already rejected by the gate, but `canonical_to_dsl` should defensively + error too). +- Inline tests in each command: `--schema` happy path, schema-validation + failure, conflicting `mdvs.toml`+`--schema` on `init` (decide policy: error or + override). **Cleanup task while wiring the gate:** -- Step 4 left a catch-all `_ =>` arm in `map_validation_error` (`crates/mdvs/src/cmd/check.rs`) that buckets unrecognized `ValidationErrorKind` variants into `WrongType`. This was defensive while the gate wasn't enforced. Once `validate_mdvs_schema` rejects every disallowed-keyword schema upstream, replace the catch-all with **exhaustive matching** so the compiler catches new `jsonschema` variants in future versions. If `ValidationErrorKind` is `#[non_exhaustive]`, the catch-all becomes `_ => unreachable!("schema gate should reject this — variant: {keyword}")` instead of bucketing. -**Definition of done:** both commands accept `--schema`; no other command does; `--help` text reflects this; `canonical_to_dsl` round-trips with `dsl_to_canonical`; `map_validation_error` no longer silently buckets unknown variants. +- Step 4 left a catch-all `_ =>` arm in `map_validation_error` + (`crates/mdvs/src/cmd/check.rs`) that buckets unrecognized + `ValidationErrorKind` variants into `WrongType`. This was defensive while the + gate wasn't enforced. Once `validate_mdvs_schema` rejects every + disallowed-keyword schema upstream, replace the catch-all with **exhaustive + matching** so the compiler catches new `jsonschema` variants in future + versions. If `ValidationErrorKind` is `#[non_exhaustive]`, the catch-all + becomes + `_ => unreachable!("schema gate should reject this — variant: {keyword}")` + instead of bucketing. + +**Definition of done:** both commands accept `--schema`; no other command does; +`--help` text reflects this; `canonical_to_dsl` round-trips with +`dsl_to_canonical`; `map_validation_error` no longer silently buckets unknown +variants. ##### Step 9 — `mdvs export-schema` command -**Prereqs:** step 2. -**Files touched:** -- `crates/mdvs/src/cmd/export_schema.rs` (new) — reads `mdvs.toml`, calls `dsl_to_canonical`, emits via `serde_json::to_string_pretty` (default) or `tomljson::to_string` (when `--format toml`). +**Prereqs:** step 2. **Files touched:** + +- `crates/mdvs/src/cmd/export_schema.rs` (new) — reads `mdvs.toml`, calls + `dsl_to_canonical`, emits via `serde_json::to_string_pretty` (default) or + `tomljson::to_string` (when `--format toml`). - `crates/mdvs/src/cmd/mod.rs` — register module. -- `crates/mdvs/src/main.rs` (or wherever clap subcommands are wired) — add `ExportSchema` variant to the command enum. +- `crates/mdvs/src/main.rs` (or wherever clap subcommands are wired) — add + `ExportSchema` variant to the command enum. **Test additions:** -- Inline tests: round-trip a representative `MdvsToml` through `export-schema` → load via step 11 → translate back to DSL via `canonical_to_dsl` → assert equal to the original `MdvsToml`. -**Definition of done:** `mdvs export-schema --output-file out.json` and `--format toml` both work; output is consumable by `mdvs init --schema`. +- Inline tests: round-trip a representative `MdvsToml` through `export-schema` → + load via step 11 → translate back to DSL via `canonical_to_dsl` → assert equal + to the original `MdvsToml`. + +**Definition of done:** `mdvs export-schema --output-file out.json` and +`--format toml` both work; output is consumable by `mdvs init --schema`. ##### Step 13 — Per-file overlay synthesis (path-scoped validation) -**Prereqs:** steps 4, 5 (uses the validator) and reuses the spike from `scripts/test_path_scoped_validation.rs`. +**Prereqs:** steps 4, 5 (uses the validator) and reuses the spike from +`scripts/test_path_scoped_validation.rs`. -**Coupled with TODO-0154**: step 13 ships with the signature-keyed overlay cache from TODO-0154 in place. The naive per-file compile approach has unacceptable scaling characteristics (~10–100µs per file compile cost, accumulating noticeably above ~5k files). Implementing without the cache and patching later was rejected — better to ship the optimized form once than ship and retrofit. +**Coupled with TODO-0154**: step 13 ships with the signature-keyed overlay cache +from TODO-0154 in place. The naive per-file compile approach has unacceptable +scaling characteristics (~10–100µs per file compile cost, accumulating +noticeably above ~5k files). Implementing without the cache and patching later +was rejected — better to ship the optimized form once than ship and retrofit. **Files touched:** -- `crates/mdvs/src/schema/overlay.rs` (new) — `synthesize_overlay(canonical, file_path) -> Value` plus the `OverlayCache` struct from TODO-0154. Walks `properties`, filters by `x-mdvs.allowed`/`required` matches against the file's path; emits `{type: "object", required: [...], additionalProperties: false, properties: {...}}`. -- `crates/mdvs/src/cmd/check.rs` — replace the Rust-side `matches_any_glob` calls and `check_required_fields` pass with a single `validator.iter_errors(&frontmatter)` against the per-file (cached) overlay. The mapping (`Required → MissingRequired`, `AdditionalProperties → Disallowed`) is already in `map_validation_error`. -**Decision: replace fully, no defense-in-depth.** Dedup logic for "both layers caught the same violation" is fiddly; keeping the Rust path means the overlay is just a side channel. Either commit to the new mechanism or skip step 13. +- `crates/mdvs/src/schema/overlay.rs` (new) — + `synthesize_overlay(canonical, file_path) -> Value` plus the `OverlayCache` + struct from TODO-0154. Walks `properties`, filters by + `x-mdvs.allowed`/`required` matches against the file's path; emits + `{type: "object", required: [...], additionalProperties: false, properties: {...}}`. +- `crates/mdvs/src/cmd/check.rs` — replace the Rust-side `matches_any_glob` + calls and `check_required_fields` pass with a single + `validator.iter_errors(&frontmatter)` against the per-file (cached) overlay. + The mapping (`Required → MissingRequired`, + `AdditionalProperties → Disallowed`) is already in `map_validation_error`. + +**Decision: replace fully, no defense-in-depth.** Dedup logic for "both layers +caught the same violation" is fiddly; keeping the Rust path means the overlay is +just a side channel. Either commit to the new mechanism or skip step 13. **Test additions:** -- Inline tests in `overlay.rs`: synthesize overlays for fixture file paths against fixture `allowed`/`required` configs; assert resulting JSON Schema matches expectation. Covers allowed-only, required-only, both, neither, ignore-list fields. -- Cache hit/miss tests (TODO-0154): same signature → same Validator; different signatures → different Validators; behavior parity between cached and uncached paths. -- End-to-end: a fixture directory where path-scoped rules differ across subdirectories; `mdvs check` produces correct per-file `Disallowed` and `MissingRequired` violations. -**Definition of done:** path-scoped validation matches the current Rust-side behaviour exactly; overlay cache produces ~1 validator per directory-tree node (not per file); `cargo test`, clippy clean; `mdvs check example_kb` parity preserved. +- Inline tests in `overlay.rs`: synthesize overlays for fixture file paths + against fixture `allowed`/`required` configs; assert resulting JSON Schema + matches expectation. Covers allowed-only, required-only, both, neither, + ignore-list fields. +- Cache hit/miss tests (TODO-0154): same signature → same Validator; different + signatures → different Validators; behavior parity between cached and uncached + paths. +- End-to-end: a fixture directory where path-scoped rules differ across + subdirectories; `mdvs check` produces correct per-file `Disallowed` and + `MissingRequired` violations. + +**Definition of done:** path-scoped validation matches the current Rust-side +behaviour exactly; overlay cache produces ~1 validator per directory-tree node +(not per file); `cargo test`, clippy clean; `mdvs check example_kb` parity +preserved. ##### Step 14 — Schema content hash in build metadata -**Prereqs:** steps 2, 12. -**Files touched:** -- `crates/mdvs/src/index/storage.rs:109–159` — extend `BuildMetadata` with `schema_hash: String` field. -- `crates/mdvs/src/cmd/build.rs:863–901` — extend `detect_config_changes` to compare `schema_hash`; mismatch requires `--force`. +**Prereqs:** steps 2, 12. **Files touched:** + +- `crates/mdvs/src/index/storage.rs:109–159` — extend `BuildMetadata` with + `schema_hash: String` field. +- `crates/mdvs/src/cmd/build.rs:863–901` — extend `detect_config_changes` to + compare `schema_hash`; mismatch requires `--force`. **Test additions:** -- Inline tests: same `mdvs.toml` produces same hash; whitespace-only edit produces same canonical hash (if hashing post-translation — see open question 3); semantic edit produces different hash. -**Definition of done:** changing the schema (DSL or `--schema` override at next init) without `--force` is rejected with a clear message; `--force` works as expected. +- Inline tests: same `mdvs.toml` produces same hash; whitespace-only edit + produces same canonical hash (if hashing post-translation — see open question + 3); semantic edit produces different hash. + +**Definition of done:** changing the schema (DSL or `--schema` override at next +init) without `--force` is rejected with a clear message; `--force` works as +expected. #### Wave B closeout — supply-chain audit and dead-code cleanup -After step 14 lands and before merging Wave B to main, run a supply-chain audit on the whole workspace. New dependencies introduced in Wave B (`jsonschema`, plus its transitive graph) substantially widen the dependency surface — confirm no advisories or license/source issues have crept in. +After step 14 lands and before merging Wave B to main, run a supply-chain audit +on the whole workspace. New dependencies introduced in Wave B (`jsonschema`, +plus its transitive graph) substantially widen the dependency surface — confirm +no advisories or license/source issues have crept in. **Tasks:** -1. `cargo install --locked cargo-audit cargo-deny` (if not already installed locally — both are dev tools, not workspace deps). -2. `cargo audit` from the repo root — runs against `Cargo.lock`. Triages any RustSec advisories. Fail-the-build if anything is flagged; resolve before merge. -3. `cargo deny check` from the repo root — covers advisories, licenses, bans, and sources in one pass. Requires a `deny.toml` at the workspace root if not already present (Wave B closeout adds one if missing — minimal config: deny GPL-family on a non-GPL project, allow MIT/Apache-2.0/BSD/MPL-2.0/ISC/Unicode-DFS-2016, ban duplicate versions of the heaviest crates). -4. **Remove temporary `#![allow(dead_code)]` annotations.** Steps 2–3 introduce new modules (`schema/json_schema.rs`, `preprocess.rs`) before any caller exists; both ship with `#![allow(dead_code)]` to keep the build clean. By the time step 14 lands every symbol must be reachable, so the annotation is removable. Grep for `#!\[allow\(dead_code\)\]` under `crates/mdvs/src/` and delete every instance. If any genuinely-dead symbol remains at this point, that's a sign the wiring is incomplete — don't paper over it with the annotation. +1. `cargo install --locked cargo-audit cargo-deny` (if not already installed + locally — both are dev tools, not workspace deps). +2. `cargo audit` from the repo root — runs against `Cargo.lock`. Triages any + RustSec advisories. Fail-the-build if anything is flagged; resolve before + merge. +3. `cargo deny check` from the repo root — covers advisories, licenses, bans, + and sources in one pass. Requires a `deny.toml` at the workspace root if not + already present (Wave B closeout adds one if missing — minimal config: deny + GPL-family on a non-GPL project, allow + MIT/Apache-2.0/BSD/MPL-2.0/ISC/Unicode-DFS-2016, ban duplicate versions of + the heaviest crates). +4. **Remove temporary `#![allow(dead_code)]` annotations.** Steps 2–3 introduce + new modules (`schema/json_schema.rs`, `preprocess.rs`) before any caller + exists; both ship with `#![allow(dead_code)]` to keep the build clean. By the + time step 14 lands every symbol must be reachable, so the annotation is + removable. Grep for `#!\[allow\(dead_code\)\]` under `crates/mdvs/src/` and + delete every instance. If any genuinely-dead symbol remains at this point, + that's a sign the wiring is incomplete — don't paper over it with the + annotation. **Files touched (closeout, not a numbered step):** + - `deny.toml` (new, at workspace root) — only if not already present. -- `.github/workflows/audit.yml` (optional, deferred decision) — recurring CI run rather than one-shot. Out of scope for Wave B unless trivial to wire. -- `crates/mdvs/src/schema/json_schema.rs`, `crates/mdvs/src/preprocess.rs`, etc. — remove `#![allow(dead_code)]`. +- `.github/workflows/audit.yml` (optional, deferred decision) — recurring CI run + rather than one-shot. Out of scope for Wave B unless trivial to wire. +- `crates/mdvs/src/schema/json_schema.rs`, `crates/mdvs/src/preprocess.rs`, etc. + — remove `#![allow(dead_code)]`. -**Definition of done:** `cargo audit` and `cargo deny check` exit zero; no `#![allow(dead_code)]` remains anywhere under `crates/mdvs/src/`; `cargo build -p mdvs` clean. +**Definition of done:** `cargo audit` and `cargo deny check` exit zero; no +`#![allow(dead_code)]` remains anywhere under `crates/mdvs/src/`; +`cargo build -p mdvs` clean. #### Open scope questions These need decisions before the relevant step starts. Listed here, not answered. -1. **`example_kb/mdvs.toml` migration.** Wave B changes validation semantics (length/pattern become first-class). Sweep the running example, or accept that pre-Wave-B `mdvs.toml` files stay valid because new keywords are additive? *(blocks step 7 cleanup; step 4 doesn't care)* -2. **`additionalProperties` semantics under inference.** Inference records every observed field; per-file overlay synthesis (step 13) tightens this. Rule needed: "field seen but not constrained" — open property or rejected outside the overlay? *(blocks step 13)* -3. **Schema content hash granularity.** Hash the canonical JSON post-translation, or the raw `mdvs.toml` bytes? Different sensitivity to whitespace and key-order changes. *(blocks step 14)* -4. **`check --schema` standalone with no `mdvs.toml`.** Synthetic in-memory `MdvsToml` with defaults, or branch every command path on `Option`? *(blocks step 10)* -5. **Step 5 mapping table location.** Lift `scripts/test_violation_mapping.rs` (22 cases) into inline tests in `cmd/check.rs`, or create the first `crates/mdvs/tests/violation_mapping.rs` integration-test file? Workspace has no integration-test directory yet — step 5 either introduces the convention or sticks to inline. *(blocks step 5)* +1. **`example_kb/mdvs.toml` migration.** Wave B changes validation semantics + (length/pattern become first-class). Sweep the running example, or accept + that pre-Wave-B `mdvs.toml` files stay valid because new keywords are + additive? _(blocks step 7 cleanup; step 4 doesn't care)_ +2. **`additionalProperties` semantics under inference.** Inference records every + observed field; per-file overlay synthesis (step 13) tightens this. Rule + needed: "field seen but not constrained" — open property or rejected outside + the overlay? _(blocks step 13)_ +3. **Schema content hash granularity.** Hash the canonical JSON + post-translation, or the raw `mdvs.toml` bytes? Different sensitivity to + whitespace and key-order changes. _(blocks step 14)_ +4. **`check --schema` standalone with no `mdvs.toml`.** Synthetic in-memory + `MdvsToml` with defaults, or branch every command path on `Option`? + _(blocks step 10)_ +5. **Step 5 mapping table location.** Lift `scripts/test_violation_mapping.rs` + (22 cases) into inline tests in `cmd/check.rs`, or create the first + `crates/mdvs/tests/violation_mapping.rs` integration-test file? Workspace has + no integration-test directory yet — step 5 either introduces the convention + or sticks to inline. _(blocks step 5)_ ### Cross-wave: documentation -- Update `docs/spec/` — constraint system, validation pipeline, new commands, `tomljson` overview. -- Update mdBook pages — concepts/schema, concepts/validation, commands/check (`--schema` override + standalone), commands/init (`--schema` import), commands/export-schema. +- Update `docs/spec/` — constraint system, validation pipeline, new commands, + `tomljson` overview. +- Update mdBook pages — concepts/schema, concepts/validation, commands/check + (`--schema` override + standalone), commands/init (`--schema` import), + commands/export-schema. - Update CLAUDE.md architecture section. ## Files to create/update **Wave A:** -- `Cargo.toml` (workspace root) — rewrite as `[workspace]` with `members = ["crates/mdvs", "crates/tomljson"]` -- `crates/mdvs/Cargo.toml` — receives the current `[package]` block + dependencies + +- `Cargo.toml` (workspace root) — rewrite as `[workspace]` with + `members = ["crates/mdvs", "crates/tomljson"]` +- `crates/mdvs/Cargo.toml` — receives the current `[package]` block + + dependencies - `crates/mdvs/src/`, `crates/mdvs/skills/` — move from `src/`, `skills/` -- `crates/tomljson/Cargo.toml`, `crates/tomljson/src/lib.rs`, `crates/tomljson/src/ser.rs`, `crates/tomljson/src/de.rs`, `crates/tomljson/tests/` — new -- `crates/tomljson/README.md` — must document scope, API surface, encoding rules, why standalone/self-describing format is out of scope -- `.github/workflows/bump.yml` — update publish step to use `-p mdvs` (and later `-p tomljson`) +- `crates/tomljson/Cargo.toml`, `crates/tomljson/src/lib.rs`, + `crates/tomljson/src/ser.rs`, `crates/tomljson/src/de.rs`, + `crates/tomljson/tests/` — new +- `crates/tomljson/README.md` — must document scope, API surface, encoding + rules, why standalone/self-describing format is out of scope +- `.github/workflows/bump.yml` — update publish step to use `-p mdvs` (and later + `-p tomljson`) - `.github/workflows/release.yml` — regenerate via `dist init` - `cog.toml` — update `pre_bump_hooks` sed path to `crates/mdvs/Cargo.toml` **Wave B:** + - `crates/mdvs/Cargo.toml` — add `jsonschema`, `tomljson` (path dep) - `crates/mdvs/src/schema/json_schema.rs` — new - `crates/mdvs/src/preprocess.rs` — new - `crates/mdvs/src/schema/constraints/mod.rs` — delegate `validate_value` - `crates/mdvs/src/schema/constraints/categories.rs` — remove `validate_value` - `crates/mdvs/src/schema/constraints/range.rs` — remove `validate_value` -- `crates/mdvs/src/discover/scan.rs` — replace silent `Pod::deserialize().ok()?` with explicit error handling that surfaces unrepresentable frontmatter values -- `crates/mdvs/src/cmd/check.rs` — replace dispatch with schema validation; add `FrontmatterUnrepresentable` to violation reporting +- `crates/mdvs/src/discover/scan.rs` — replace silent `Pod::deserialize().ok()?` + with explicit error handling that surfaces unrepresentable frontmatter values +- `crates/mdvs/src/cmd/check.rs` — replace dispatch with schema validation; add + `FrontmatterUnrepresentable` to violation reporting - `crates/mdvs/src/cmd/export_schema.rs` — new - `crates/mdvs/src/cmd/init.rs` — add `--schema` (one-shot import) -- `crates/mdvs/src/cmd/check.rs` — add `--schema` (read-only override; allow `mdvs.toml` to be absent with defaults) +- `crates/mdvs/src/cmd/check.rs` — add `--schema` (read-only override; allow + `mdvs.toml` to be absent with defaults) - `crates/mdvs/src/cmd/init.rs` — add `--schema` (one-shot import) -- `crates/mdvs/src/schema/load.rs` (or similar) — file-extension-sniffing schema loader (`.json` / `.toml`) +- `crates/mdvs/src/schema/load.rs` (or similar) — file-extension-sniffing schema + loader (`.json` / `.toml`) - `crates/mdvs/src/schema/config.rs` — add `min_length`, `max_length`, `pattern` -- `book/src/concepts/validation.md` — document the YAML→JSON conversion rules and what causes `FrontmatterUnrepresentable` violations +- `book/src/concepts/validation.md` — document the YAML→JSON conversion rules + and what causes `FrontmatterUnrepresentable` violations **Wave C:** + - `crates/mdvs/src/discover/field_type.rs` — rename variants - `crates/mdvs/src/schema/config.rs` — flattened object representation - `example_kb/mdvs.toml` — migrate @@ -1633,17 +2300,25 @@ These need decisions before the relevant step starts. Listed here, not answered. ## Further crate splits (out of scope, no commitment) -Wave A extracts `tomljson`. Beyond that, **no further splits are planned in TODO-0149**. The workspace stays at two crates: `tomljson` and `mdvs`. +Wave A extracts `tomljson`. Beyond that, **no further splits are planned in +TODO-0149**. The workspace stays at two crates: `tomljson` and `mdvs`. -A future `mdvs-validate` / `mdvs-search` split has been discussed and discarded for now — there is no second consumer of either half, scan/parse/config/output are needed by both, and committing a public API across the seam pre-1.0 would lock in choices that Wave B and C may want to reshape. Revisit only when a concrete trigger fires (external library consumer, binary-size pressure, or a server/IDE/MCP integration that wants validation alone). +A future `mdvs-validate` / `mdvs-search` split has been discussed and discarded +for now — there is no second consumer of either half, scan/parse/config/output +are needed by both, and committing a public API across the seam pre-1.0 would +lock in choices that Wave B and C may want to reshape. Revisit only when a +concrete trigger fires (external library consumer, binary-size pressure, or a +server/IDE/MCP integration that wants validation alone). ## Related -- TODO-0006 (categories) — done, `validate_value` migrated to jsonschema in Wave B +- TODO-0006 (categories) — done, `validate_value` migrated to jsonschema in Wave + B - TODO-0008 (numeric boundaries) — subsumed - TODO-0010 (length constraints) — subsumed - TODO-0145 (regex pattern) — subsumed - TODO-0097 (flatten nested objects) — converges into Wave C -- TODO-0143 (additional constraint kinds) — future constraints become new JSON Schema keywords +- TODO-0143 (additional constraint kinds) — future constraints become new JSON + Schema keywords - TODO-0144 (Lua scripting) — custom preprocessors plug into the pipeline - TODO-0150 (strict mode) — `--strict` skips preprocessors diff --git a/docs/spec/todos/TODO-0150.md b/docs/spec/todos/TODO-0150.md index c67eccf..689d134 100644 --- a/docs/spec/todos/TODO-0150.md +++ b/docs/spec/todos/TODO-0150.md @@ -12,65 +12,102 @@ blocks: [] ## Summary -Add an opt-in strict mode that disables mdvs's pre-coercion leniencies. In strict mode, inference errors on mixed types instead of widening, and validation rejects `1` in a String field, `"1"` in an Integer field, and an integer in a Float field. This gives users with contractual data requirements (Pydantic-style) the ability to catch drift that lenient mode silently accepts. +Add an opt-in strict mode that disables mdvs's pre-coercion leniencies. In +strict mode, inference errors on mixed types instead of widening, and validation +rejects `1` in a String field, `"1"` in an Integer field, and an integer in a +Float field. This gives users with contractual data requirements +(Pydantic-style) the ability to catch drift that lenient mode silently accepts. ## Motivation -mdvs's default lenient behavior is correct for its primary audience — informal markdown vaults (Obsidian, personal notes) where users don't think about `1` vs `"1"`. Type widening and post-widening acceptance preserve inference-to-validation consistency. +mdvs's default lenient behavior is correct for its primary audience — informal +markdown vaults (Obsidian, personal notes) where users don't think about `1` vs +`"1"`. Type widening and post-widening acceptance preserve +inference-to-validation consistency. -But a second audience exists: **structured data used in CI, typed pipelines, or compliance contexts**. For them, `priority: 1` and `priority: "1"` are different values that should round-trip identically through the schema. Silent widening hides bugs. +But a second audience exists: **structured data used in CI, typed pipelines, or +compliance contexts**. For them, `priority: 1` and `priority: "1"` are different +values that should round-trip identically through the schema. Silent widening +hides bugs. -Pydantic precedent: `strict=False` (default) allows coercion; `strict=True` enforces exact types. Both are legitimate contracts for the same tool. +Pydantic precedent: `strict=False` (default) allows coercion; `strict=True` +enforces exact types. Both are legitimate contracts for the same tool. ## Behavior ### Strict inference -- A field with mixed types across files → **error**, not widening. The `init` or `update` command reports the conflict and refuses to write the schema for that field. -- User's options: fix the data (quote consistently), add the field to `[fields].ignore`, or manually author the schema. -- Changes `init` from infallible to fallible. The error message should pinpoint which files caused the conflict. +- A field with mixed types across files → **error**, not widening. The `init` or + `update` command reports the conflict and refuses to write the schema for that + field. +- User's options: fix the data (quote consistently), add the field to + `[fields].ignore`, or manually author the schema. +- Changes `init` from infallible to fallible. The error message should pinpoint + which files caused the conflict. ### Strict validation -- **String field with integer value** → `WrongType` violation. (Currently silently accepted.) -- **Integer field with string value** → `WrongType` violation. (Already fires in lenient mode for most cases; confirm this is consistent in strict mode.) -- **Float field with integer value** → `WrongType` violation. (Currently accepted.) Edge case to decide: YAML can't distinguish `5` from `5.0`; the user writing `5` genuinely may mean `5.0`. Options: +- **String field with integer value** → `WrongType` violation. (Currently + silently accepted.) +- **Integer field with string value** → `WrongType` violation. (Already fires in + lenient mode for most cases; confirm this is consistent in strict mode.) +- **Float field with integer value** → `WrongType` violation. (Currently + accepted.) Edge case to decide: YAML can't distinguish `5` from `5.0`; the + user writing `5` genuinely may mean `5.0`. Options: - Reject — truly strict, user must write `5.0`. - - Accept — keep this one leniency because it's YAML-level, not mdvs-level. Pydantic accepts integers in float fields even in strict mode for similar reasons. + - Accept — keep this one leniency because it's YAML-level, not mdvs-level. + Pydantic accepts integers in float fields even in strict mode for similar + reasons. -Recommend: accept integers in Float fields even in strict mode (YAML limitation, not a semantic choice). Document the nuance. +Recommend: accept integers in Float fields even in strict mode (YAML limitation, +not a semantic choice). Document the nuance. ## Scope Start global + CLI override, defer per-field: - **Config**: `[fields].strict = true` in `mdvs.toml` (default `false`) -- **CLI override**: `mdvs check --strict` for one-shot enforcement without touching config +- **CLI override**: `mdvs check --strict` for one-shot enforcement without + touching config - **Per-field strict mode**: deferred until someone asks for it -The `--strict` flag on `check` and `build` makes CI usage painless without changing the committed config. +The `--strict` flag on `check` and `build` makes CI usage painless without +changing the committed config. ## Implementation notes -This is straightforward once TODO-0149 lands. In the JSON-Schema-delegation architecture: +This is straightforward once TODO-0149 lands. In the JSON-Schema-delegation +architecture: -- **Lenient mode (default)**: mdvs pre-coerces values before delegating to `jsonschema` (preserves current behavior). -- **Strict mode**: skip the pre-coercion step. Pass raw values directly. The library's strict errors bubble up through our wrapping. +- **Lenient mode (default)**: mdvs pre-coerces values before delegating to + `jsonschema` (preserves current behavior). +- **Strict mode**: skip the pre-coercion step. Pass raw values directly. The + library's strict errors bubble up through our wrapping. Essentially "delete the coercion step" in strict mode. Minimal code. -Inference is the larger change — `collect_types` becomes fallible under strict mode, reporting per-field conflicts with the offending file paths. +Inference is the larger change — `collect_types` becomes fallible under strict +mode, reporting per-field conflicts with the offending file paths. ## Open questions -1. **Default**: lenient (current behavior) or strict? Recommend: keep lenient as default. Strict is opt-in. -2. **Float accepts Integer even in strict mode?** Recommend: yes, YAML-level leniency is separate from mdvs-level. Document clearly. -3. **Config location**: `[fields].strict` or a dedicated `[validation].strict` section? The latter is more extensible if strict mode grows sub-settings. -4. **Inference errors** — how to present them? Init/update currently returns a single `UpdateOutcome`; we'd add a `conflicts` list with per-field, per-file details. -5. **JSON Schema export in strict mode** — the exported schema is the same; strict mode is a mdvs-validator behavior, not a schema property. Third-party tools already do strict validation when consuming JSON Schema. +1. **Default**: lenient (current behavior) or strict? Recommend: keep lenient as + default. Strict is opt-in. +2. **Float accepts Integer even in strict mode?** Recommend: yes, YAML-level + leniency is separate from mdvs-level. Document clearly. +3. **Config location**: `[fields].strict` or a dedicated `[validation].strict` + section? The latter is more extensible if strict mode grows sub-settings. +4. **Inference errors** — how to present them? Init/update currently returns a + single `UpdateOutcome`; we'd add a `conflicts` list with per-field, per-file + details. +5. **JSON Schema export in strict mode** — the exported schema is the same; + strict mode is a mdvs-validator behavior, not a schema property. Third-party + tools already do strict validation when consuming JSON Schema. ## Related - TODO-0149 (JSON Schema delegation) — makes strict mode trivial to add -- TODO-0006 (categories), TODO-0008 (range) — current validators that would be affected -- MEMORY.md — documents the current "String is top type" and "Integer-for-Float" rules as inference-correctness features +- TODO-0006 (categories), TODO-0008 (range) — current validators that would be + affected +- MEMORY.md — documents the current "String is top type" and "Integer-for-Float" + rules as inference-correctness features diff --git a/docs/spec/todos/TODO-0151.md b/docs/spec/todos/TODO-0151.md index aa10d5c..3a509aa 100644 --- a/docs/spec/todos/TODO-0151.md +++ b/docs/spec/todos/TODO-0151.md @@ -13,7 +13,13 @@ blocks: [] ## Summary -When a field has both string and array values across files (e.g., `funding: "internal"` and `funding: ["internal"]`), the type widens to `String`. Category inference and validation are inconsistent with this widening: inference collects distinct values at element level (extracting array elements), while validation compares the raw JSON array against string categories. This causes `check` to fail immediately after `init` on the same files — violating the invariant that inference output must pass validation. +When a field has both string and array values across files (e.g., +`funding: "internal"` and `funding: ["internal"]`), the type widens to `String`. +Category inference and validation are inconsistent with this widening: inference +collects distinct values at element level (extracting array elements), while +validation compares the raw JSON array against string categories. This causes +`check` to fail immediately after `init` on the same files — violating the +invariant that inference output must pass validation. ## Details @@ -21,33 +27,57 @@ When a field has both string and array values across files (e.g., `funding: "int Two layers of inconsistency when a field widens from `Array(T)` to `String`: -1. **`collect_distinct_values` (`discover/infer/types.rs`)**: Always expands arrays to their elements. From `["internal"]` it extracts `"internal"`, making it look identical to the plain string `"internal"`. The distinct values don't reflect what would actually be stored — build would serialize the array to `"[\"internal\"]"`, a different string. +1. **`collect_distinct_values` (`discover/infer/types.rs`)**: Always expands + arrays to their elements. From `["internal"]` it extracts `"internal"`, + making it look identical to the plain string `"internal"`. The distinct + values don't reflect what would actually be stored — build would serialize + the array to `"[\"internal\"]"`, a different string. -2. **`validate_value` (`schema/constraints/categories.rs`)**: For `FieldType::String`, compares the raw frontmatter value (a JSON array) against string categories. `toml_json_eq(String("internal"), Array(["internal"]))` always returns `false`. +2. **`validate_value` (`schema/constraints/categories.rs`)**: For + `FieldType::String`, compares the raw frontmatter value (a JSON array) + against string categories. + `toml_json_eq(String("internal"), Array(["internal"]))` always returns + `false`. ### Fix -Both layers need fixing, without adding a second pass over files (to avoid coupling to the current "everything in memory" design of `ScannedFiles`): +Both layers need fixing, without adding a second pass over files (to avoid +coupling to the current "everything in memory" design of `ScannedFiles`): -1. **Collection (`collect_distinct_values`)**: Stop expanding arrays during collection. Always add the raw value. After the type-widening loop completes, post-process `distinct_values` per field based on the final widened type: +1. **Collection (`collect_distinct_values`)**: Stop expanding arrays during + collection. Always add the raw value. After the type-widening loop completes, + post-process `distinct_values` per field based on the final widened type: - `Array(T)` field → expand arrays to elements, deduplicate - - `String` field → serialize any non-string/non-null values with `Value::to_string()` (e.g., `["internal"]` becomes the string `["internal"]`) - - Other scalar fields → no change (arrays can't widen to Integer/Float/Boolean) + - `String` field → serialize any non-string/non-null values with + `Value::to_string()` (e.g., `["internal"]` becomes the string + `["internal"]`) + - Other scalar fields → no change (arrays can't widen to + Integer/Float/Boolean) - Occurrence counts also need adjusting: during collection, count arrays as 1 occurrence (the whole value); for `Array(T)` fields, re-count at element level during post-processing. + Occurrence counts also need adjusting: during collection, count arrays as 1 + occurrence (the whole value); for `Array(T)` fields, re-count at element + level during post-processing. -2. **Validation (`validate_value`)**: In the `String | Integer` branch, when the raw frontmatter value doesn't match the field type shape (e.g., a JSON array in a `String` field), serialize it with `Value::to_string()` before comparing against categories — matching what `build` would store. +2. **Validation (`validate_value`)**: In the `String | Integer` branch, when the + raw frontmatter value doesn't match the field type shape (e.g., a JSON array + in a `String` field), serialize it with `Value::to_string()` before comparing + against categories — matching what `build` would store. ### Reproduction `example_kb` has a `funding` field that triggers this: + - `projects/alpha/notes/experiment-{1,2,3}.md`: `funding: "internal"` (string) -- `projects/archived/gamma/{post-mortem,lessons-learned}.md`: `funding: "internal"` (string) -- `projects/beta/notes/{initial-findings,replication}.md`: `funding: ["internal"]` (array) +- `projects/archived/gamma/{post-mortem,lessons-learned}.md`: + `funding: "internal"` (string) +- `projects/beta/notes/{initial-findings,replication}.md`: + `funding: ["internal"]` (array) -`mdvs init --force example_kb && mdvs check example_kb` → 1 violation (`Invalid category` on the 2 beta files). +`mdvs init --force example_kb && mdvs check example_kb` → 1 violation +(`Invalid category` on the 2 beta files). -Also reproduced on the Refractions vault (`projects` field: mix of string and array values across 89 files). +Also reproduced on the Refractions vault (`projects` field: mix of string and +array values across 89 files). ### Files to update diff --git a/docs/spec/todos/TODO-0152.md b/docs/spec/todos/TODO-0152.md index 692b5a5..65b0ec6 100644 --- a/docs/spec/todos/TODO-0152.md +++ b/docs/spec/todos/TODO-0152.md @@ -12,29 +12,60 @@ blocks: [] ## Summary -Reorganize `crates/mdvs/src/` into four explicit layers — **validate engine**, **ingest**, **search**, **cli** — with a strict rule that each layer imports only from layers above it. The validate engine becomes a self-contained module with a closed contract: schema in, tagged instances in, violations out — no filesystem, no markdown, no YAML knowledge. This makes the validation kernel extractable as a standalone crate when a trigger fires (TODO-0149's discarded crate-split discussion captures the triggers), and clarifies the dependency graph for everything else. +Reorganize `crates/mdvs/src/` into four explicit layers — **validate engine**, +**ingest**, **search**, **cli** — with a strict rule that each layer imports +only from layers above it. The validate engine becomes a self-contained module +with a closed contract: schema in, tagged instances in, violations out — no +filesystem, no markdown, no YAML knowledge. This makes the validation kernel +extractable as a standalone crate when a trigger fires (TODO-0149's discarded +crate-split discussion captures the triggers), and clarifies the dependency +graph for everything else. ## Motivation -The structural concern that surfaced during TODO-0149 design: **the lines between mdvs's modules are blurred.** Several specific smells: - -- `cmd/check.rs` holds the `validate()` function — domain logic depends on the CLI layer, not the other way around. `build` calls into `cmd/check.rs` to validate, which entangles two CLI-level files in a domain operation. -- `output.rs` is a grab-bag: `ViolationKind` (validation domain), `DiscoveredField` (discovery domain), `BuildFileDetail` (build domain), `CommandOutput` trait (orchestration). Four layers in one file. -- `schema/` mixes config types, translation logic, and (after Wave B) jsonschema dispatch + partition + error mapping. Risk: 2000+ line module without internal structure. -- `index/backend.rs` was a half-built abstraction in the Parquet+DataFusion era — used only for writes/reads from `storage.rs` while `search.rs` bypassed it with direct DataFusion calls. After the Lance + LanceDB swap (TODO-0016, shipped 2026-05-25), `backend.rs` now also owns the search path (mode dispatch, `--where` translator, result assembly), so the file has grown without the layered shape that would make extraction natural. +The structural concern that surfaced during TODO-0149 design: **the lines +between mdvs's modules are blurred.** Several specific smells: + +- `cmd/check.rs` holds the `validate()` function — domain logic depends on the + CLI layer, not the other way around. `build` calls into `cmd/check.rs` to + validate, which entangles two CLI-level files in a domain operation. +- `output.rs` is a grab-bag: `ViolationKind` (validation domain), + `DiscoveredField` (discovery domain), `BuildFileDetail` (build domain), + `CommandOutput` trait (orchestration). Four layers in one file. +- `schema/` mixes config types, translation logic, and (after Wave B) jsonschema + dispatch + partition + error mapping. Risk: 2000+ line module without internal + structure. +- `index/backend.rs` was a half-built abstraction in the Parquet+DataFusion era + — used only for writes/reads from `storage.rs` while `search.rs` bypassed it + with direct DataFusion calls. After the Lance + LanceDB swap (TODO-0016, + shipped 2026-05-25), `backend.rs` now also owns the search path (mode + dispatch, `--where` translator, result assembly), so the file has grown + without the layered shape that would make extraction natural. - No place for the preprocessor pipeline introduced by TODO-0149. -The deeper diagnosis: the codebase is grouped by **what the data is** (`discover/`, `schema/`, `index/`) rather than by **what we do with it**. That made sense at the start; it strains as the surface grows and as Wave B/C and the Lance + LanceDB swap add more weight. +The deeper diagnosis: the codebase is grouped by **what the data is** +(`discover/`, `schema/`, `index/`) rather than by **what we do with it**. That +made sense at the start; it strains as the surface grows and as Wave B/C and the +Lance + LanceDB swap add more weight. ## Goal: a self-contained validate engine -Apply the **`tomljson` test** to every part of mdvs: which pieces have a closed contract that needs no implicit context from the rest of the codebase? Most of mdvs is integration glue around well-known crates (walkdir, gray_matter, text-splitter, model2vec, lancedb). The genuinely novel kernel is: +Apply the **`tomljson` test** to every part of mdvs: which pieces have a closed +contract that needs no implicit context from the rest of the codebase? Most of +mdvs is integration glue around well-known crates (walkdir, gray_matter, +text-splitter, model2vec, lancedb). The genuinely novel kernel is: -- JSON Schema validation with **`x-mdvs` path-scoping** (per-context presence rules). -- A **three-stage preprocessor pipeline** (field-name, per-field, per-document) with built-in variants and a Lua escape hatch. -- A **constraint-aware error mapper** that translates `jsonschema::ValidationError` into actionable violations. +- JSON Schema validation with **`x-mdvs` path-scoping** (per-context presence + rules). +- A **three-stage preprocessor pipeline** (field-name, per-field, per-document) + with built-in variants and a Lua escape hatch. +- A **constraint-aware error mapper** that translates + `jsonschema::ValidationError` into actionable violations. -These are mdvs's intellectual property. Every other piece is a wrapper. The validate engine should be packaged so that someone could (eventually) take it standalone and use it to validate any tagged JSON instance set against any compatible schema — markdown frontmatters happen to be one application. +These are mdvs's intellectual property. Every other piece is a wrapper. The +validate engine should be packaged so that someone could (eventually) take it +standalone and use it to validate any tagged JSON instance set against any +compatible schema — markdown frontmatters happen to be one application. ## Proposed structure @@ -94,22 +125,29 @@ crates/mdvs/src/ cli → search → ingest → validate → tomljson (workspace dep) ``` -Imports flow **downward only**. No back-edges. If a back-edge becomes necessary, it points at a missing abstraction that needs to be hoisted to a lower layer. +Imports flow **downward only**. No back-edges. If a back-edge becomes necessary, +it points at a missing abstraction that needs to be hoisted to a lower layer. Examples of what this rule disallows: -- `validate/` importing `serde_yaml` or `gray_matter` types — the engine doesn't know YAML exists. +- `validate/` importing `serde_yaml` or `gray_matter` types — the engine doesn't + know YAML exists. - `validate/` importing from `ingest/` — instances arrive pre-extracted. - `ingest/` importing from `search/` — ingest produces, search consumes. -- `search/` importing from `cli/` — CLI orchestrates, doesn't get called by search. -- `output/` types appearing in domain structs — `ViolationKind` is a domain type; `format_text` for it lives in `cli/output/`. +- `search/` importing from `cli/` — CLI orchestrates, doesn't get called by + search. +- `output/` types appearing in domain structs — `ViolationKind` is a domain + type; `format_text` for it lives in `cli/output/`. Examples of what becomes possible: -- The validate engine has unit tests fed by hand-crafted JSON instances. No filesystem, no markdown. +- The validate engine has unit tests fed by hand-crafted JSON instances. No + filesystem, no markdown. - The ingest layer has unit tests fed by an in-memory directory. No validation. -- A future `cargo test --package mdvs --features validate-only` could compile only the validate layer for fast CI. -- The validate engine can be moved to `crates/mdvs-validate/` later by literally relocating the directory — no logic changes. +- A future `cargo test --package mdvs --features validate-only` could compile + only the validate layer for fast CI. +- The validate engine can be moved to `crates/mdvs-validate/` later by literally + relocating the directory — no logic changes. ## Closed contract for the validate engine @@ -145,58 +183,105 @@ pub struct Violation { } ``` -**No filesystem.** **No markdown.** **No YAML.** **No `MdvsToml` struct in the public API** — the DSL ↔ canonical translator is internal; consumers hand in canonical JSON Schema. (Inside `validate/schema/dsl.rs` the DSL types still exist for serde-driven config loading, but they're not part of the engine's input contract.) +**No filesystem.** **No markdown.** **No YAML.** **No `MdvsToml` struct in the +public API** — the DSL ↔ canonical translator is internal; consumers hand in +canonical JSON Schema. (Inside `validate/schema/dsl.rs` the DSL types still +exist for serde-driven config loading, but they're not part of the engine's +input contract.) -This is the contract that makes the engine extractable. mdvs's CLI uses it with filepath as the context tag; another consumer might use environment names, tenant IDs, or document UUIDs. +This is the contract that makes the engine extractable. mdvs's CLI uses it with +filepath as the context tag; another consumer might use environment names, +tenant IDs, or document UUIDs. ## What this is NOT -- **Not** a separate crate yet. The reorganization is internal to `crates/mdvs/`. Extraction to `crates/mdvs-validate/` is a separate future decision tied to the triggers documented in TODO-0149's "Further crate splits" section. -- **Not** a public API stabilization. The engine's API can churn freely as long as it stays internal. The discipline is purely import direction, not semver. +- **Not** a separate crate yet. The reorganization is internal to + `crates/mdvs/`. Extraction to `crates/mdvs-validate/` is a separate future + decision tied to the triggers documented in TODO-0149's "Further crate splits" + section. +- **Not** a public API stabilization. The engine's API can churn freely as long + as it stays internal. The discipline is purely import direction, not semver. - **Not** a workspace restructure (TODO-0149 Wave A already does that). - **Not** a rewrite. Existing logic moves to new files; behavior unchanged. ## Acceptance criteria -- [ ] `crates/mdvs/src/` reorganized into `validate/`, `ingest/`, `search/`, `cli/`. +- [ ] `crates/mdvs/src/` reorganized into `validate/`, `ingest/`, `search/`, + `cli/`. - [ ] `cargo build` and `cargo test` pass throughout the migration. -- [ ] No back-edges in the dependency graph. Verifiable via `cargo-modules` or `cargo-deps` snapshot. -- [ ] `validate/` does not import `walkdir`, `gray_matter`, `serde_yaml`, `pulldown_cmark`, or any path-related crate. -- [ ] `validate/` has unit tests that construct `ValidateEngine` from a hand-built `serde_json::Value` and exercise the partition + overlay + preprocess + jsonschema chain end-to-end without touching the filesystem. +- [ ] No back-edges in the dependency graph. Verifiable via `cargo-modules` or + `cargo-deps` snapshot. +- [ ] `validate/` does not import `walkdir`, `gray_matter`, `serde_yaml`, + `pulldown_cmark`, or any path-related crate. +- [ ] `validate/` has unit tests that construct `ValidateEngine` from a + hand-built `serde_json::Value` and exercise the partition + overlay + + preprocess + jsonschema chain end-to-end without touching the filesystem. - [ ] `ingest/` does not import from `validate/` or `search/`. - [ ] `cli/` is the only place that imports `clap`. -- [ ] `output.rs` is gone — its contents are split between `validate/violation.rs` (domain types) and `cli/output/` (formatters). -- [ ] `cmd/check.rs` no longer holds `validate()` — it orchestrates an `ingest` stream into a `ValidateEngine`. -- [ ] All existing CLI behavior unchanged (verified via existing test suite + manual smoke against `example_kb`). +- [ ] `output.rs` is gone — its contents are split between + `validate/violation.rs` (domain types) and `cli/output/` (formatters). +- [ ] `cmd/check.rs` no longer holds `validate()` — it orchestrates an `ingest` + stream into a `ValidateEngine`. +- [ ] All existing CLI behavior unchanged (verified via existing test suite + + manual smoke against `example_kb`). ## Order of work -1. **Stage 0 — landmark check**: lock in current dependency graph as a baseline so we can detect new back-edges introduced during the move. `cargo-modules generate graph > before.dot`. -2. **Stage 1 — `validate/` materializes**: move `schema/`, `output.rs::ViolationKind` + related, the partition/overlay/preprocess/error-map code introduced by TODO-0149 Wave B. Land alongside Wave B so the engine is born layered, not retrofitted. -3. **Stage 2 — `ingest/` materializes**: move `discover/scan.rs`, `discover/field_type.rs` (the parts that don't belong to inference), introduce `yaml2json.rs` as the explicit conversion module fixing TODO-0149's silent `.ok()?`. -4. **Stage 3 — `search/` materializes**: move `index/`, `search.rs`. Split the post-TODO-0016 `backend.rs` (which now mixes storage, mode dispatch, and `--where` translation) into `storage/lance.rs`, `mode.rs`, and `query.rs`. -5. **Stage 4 — `cli/` consolidates**: move `cmd/`, output formatting, telemetry/`tracing` setup. Delete `output.rs` (its types went to `validate/violation.rs` in stage 1; its formatters land here). -6. **Stage 5 — verify the rule**: run `cargo-modules generate graph > after.dot` and visually confirm the dependency graph is a strict DAG with the expected layering. Add a CI check (custom script or `cargo-deny` rule) to prevent regressions. +1. **Stage 0 — landmark check**: lock in current dependency graph as a baseline + so we can detect new back-edges introduced during the move. + `cargo-modules generate graph > before.dot`. +2. **Stage 1 — `validate/` materializes**: move `schema/`, + `output.rs::ViolationKind` + related, the + partition/overlay/preprocess/error-map code introduced by TODO-0149 Wave B. + Land alongside Wave B so the engine is born layered, not retrofitted. +3. **Stage 2 — `ingest/` materializes**: move `discover/scan.rs`, + `discover/field_type.rs` (the parts that don't belong to inference), + introduce `yaml2json.rs` as the explicit conversion module fixing TODO-0149's + silent `.ok()?`. +4. **Stage 3 — `search/` materializes**: move `index/`, `search.rs`. Split the + post-TODO-0016 `backend.rs` (which now mixes storage, mode dispatch, and + `--where` translation) into `storage/lance.rs`, `mode.rs`, and `query.rs`. +5. **Stage 4 — `cli/` consolidates**: move `cmd/`, output formatting, + telemetry/`tracing` setup. Delete `output.rs` (its types went to + `validate/violation.rs` in stage 1; its formatters land here). +6. **Stage 5 — verify the rule**: run `cargo-modules generate graph > after.dot` + and visually confirm the dependency graph is a strict DAG with the expected + layering. Add a CI check (custom script or `cargo-deny` rule) to prevent + regressions. ## Dependency on TODO-0149 This TODO depends on TODO-0149 because: -- Wave A's workspace restructure must land first; this reorganization happens inside `crates/mdvs/src/` after the workspace exists. -- Wave B introduces the validate engine's actual content (partition, overlay, preprocessor pipeline, jsonschema integration). Reorganizing before Wave B would shuffle modules that are about to be replaced. -- Wave C's type renames and object flattening are the last destabilizing move on the schema types; doing the layered reorg after Wave C means we move stable code, not soon-to-be-renamed code. +- Wave A's workspace restructure must land first; this reorganization happens + inside `crates/mdvs/src/` after the workspace exists. +- Wave B introduces the validate engine's actual content (partition, overlay, + preprocessor pipeline, jsonschema integration). Reorganizing before Wave B + would shuffle modules that are about to be replaced. +- Wave C's type renames and object flattening are the last destabilizing move on + the schema types; doing the layered reorg after Wave C means we move stable + code, not soon-to-be-renamed code. -So the correct sequence is: **TODO-0149 Wave A → Wave B → Wave C → TODO-0152**. Or, alternatively, **TODO-0152's Stage 1 lands together with Wave B** if we want the validate engine to be born layered (recommended). +So the correct sequence is: **TODO-0149 Wave A → Wave B → Wave C → TODO-0152**. +Or, alternatively, **TODO-0152's Stage 1 lands together with Wave B** if we want +the validate engine to be born layered (recommended). ## Out of scope -- Extraction to a separate `mdvs-validate` crate. Captured as a future trigger in TODO-0149. +- Extraction to a separate `mdvs-validate` crate. Captured as a future trigger + in TODO-0149. - Changes to public CLI behavior, output format, or `mdvs.toml` shape. -- LanceDB migration — already shipped via TODO-0016 (2026-05-25). This reorganization moves the post-swap `index/backend.rs` (storage + search) into the `search/` layer; no functional change. +- LanceDB migration — already shipped via TODO-0016 (2026-05-25). This + reorganization moves the post-swap `index/backend.rs` (storage + search) into + the `search/` layer; no functional change. ## Related -- TODO-0149 — JSON Schema refoundation; the layered structure is the natural home for its outputs. +- TODO-0149 — JSON Schema refoundation; the layered structure is the natural + home for its outputs. - TODO-0144 — Lua preprocessors land in `validate/preprocess/lua.rs`. -- TODO-0016 — shipped 2026-05-25. The post-swap `index/backend.rs` (LanceBackend + mode dispatch + `--where` translator) is the largest single file that this reorganization would split. -- TODO-0079, TODO-0080, TODO-0119 — earlier pipeline-abstraction work (Step tree). This TODO is a structural reorganization above that, not a redo of it. +- TODO-0016 — shipped 2026-05-25. The post-swap `index/backend.rs` + (LanceBackend + mode dispatch + `--where` translator) is the largest single + file that this reorganization would split. +- TODO-0079, TODO-0080, TODO-0119 — earlier pipeline-abstraction work (Step + tree). This TODO is a structural reorganization above that, not a redo of it. diff --git a/docs/spec/todos/TODO-0153.md b/docs/spec/todos/TODO-0153.md index d831cc6..6621198 100644 --- a/docs/spec/todos/TODO-0153.md +++ b/docs/spec/todos/TODO-0153.md @@ -12,54 +12,92 @@ blocks: [] ## Summary -Captures the design context for a hypothetical future set of **small focused Rust crates** that fill specific gaps in `jsonschema` and that mdvs would consume rather than building inline. Concretely: one crate adds context-aware custom keywords, one adds a coerce-before-validate step, one adds a structured preprocessor pipeline. Each is composable, small, and stands alone. mdvs would consume all three; other Rust projects would pick whichever they need. **Deferred** — captured here as design context, not a commitment. - -This TODO replaces an earlier framing that proposed a single bundled "validation engine" crate. That bundled framing was rejected because it doesn't fit Rust ecosystem culture (Rust devs prefer composable small crates over framework-shaped bundles). +Captures the design context for a hypothetical future set of **small focused +Rust crates** that fill specific gaps in `jsonschema` and that mdvs would +consume rather than building inline. Concretely: one crate adds context-aware +custom keywords, one adds a coerce-before-validate step, one adds a structured +preprocessor pipeline. Each is composable, small, and stands alone. mdvs would +consume all three; other Rust projects would pick whichever they need. +**Deferred** — captured here as design context, not a commitment. + +This TODO replaces an earlier framing that proposed a single bundled "validation +engine" crate. That bundled framing was rejected because it doesn't fit Rust +ecosystem culture (Rust devs prefer composable small crates over +framework-shaped bundles). ## Why this exists as a TODO -During TODO-0149's design discussion, we asked whether the validation pipeline mdvs is building (`jsonschema` + path-scoping + preprocessors + Lua) is novel enough to extract as a standalone crate. The conversation went through several reframings: - -1. **First framing**: "extract the whole pipeline as one engine crate." Rejected — the engine isn't novel enough to compete with existing options as a bundle, and Rust devs don't reach for bundles anyway. -2. **Second framing** ("Pydantic for Rust"): rejected — wrong audience signal, wrong selling point. Rust devs don't pick libraries for ergonomics, and Pydantic's name carries a Python-shaped expectation that doesn't translate. -3. **Third framing** (this TODO): three small composable crates each filling a specific gap in `jsonschema`. Aligns with Rust culture: small crates, clear purpose, opt-in adoption, compose-as-needed. mdvs uses all three; other projects pick whatever they need. - -The realization that enabled the split: mdvs's path-scoping isn't novel at runtime — it's a glob match against a string in a context bag, which is itself just a generic mechanism that any consumer might want. The mdvs novelty about paths is in the **inference algorithm** (`DirectoryTree`, `GlobMap`, glob-collapse), not the runtime. Once you accept that, the validation pipeline naturally decomposes into small reusable pieces. +During TODO-0149's design discussion, we asked whether the validation pipeline +mdvs is building (`jsonschema` + path-scoping + preprocessors + Lua) is novel +enough to extract as a standalone crate. The conversation went through several +reframings: + +1. **First framing**: "extract the whole pipeline as one engine crate." Rejected + — the engine isn't novel enough to compete with existing options as a bundle, + and Rust devs don't reach for bundles anyway. +2. **Second framing** ("Pydantic for Rust"): rejected — wrong audience signal, + wrong selling point. Rust devs don't pick libraries for ergonomics, and + Pydantic's name carries a Python-shaped expectation that doesn't translate. +3. **Third framing** (this TODO): three small composable crates each filling a + specific gap in `jsonschema`. Aligns with Rust culture: small crates, clear + purpose, opt-in adoption, compose-as-needed. mdvs uses all three; other + projects pick whatever they need. + +The realization that enabled the split: mdvs's path-scoping isn't novel at +runtime — it's a glob match against a string in a context bag, which is itself +just a generic mechanism that any consumer might want. The mdvs novelty about +paths is in the **inference algorithm** (`DirectoryTree`, `GlobMap`, +glob-collapse), not the runtime. Once you accept that, the validation pipeline +naturally decomposes into small reusable pieces. ## The Rust ecosystem audit The Rust validation ecosystem splits into three camps: -| Approach | Crates | What it covers | Gap from this TODO | -|---|---|---|---| -| **Schema-as-Rust-type** | `validator`, `garde`, `nutype`, `serde` + custom impls | known-at-compile-time schemas, derived from struct definitions | wrong shape — schema is code, not data | -| **Schema-as-data, validation-only** | `jsonschema` | JSON Schema 2020-12 validation, custom keywords supported, no first-class context, no coercion, no preprocessor pipeline | the foundation we'd build *on top of* | -| **Schema-as-data, validation + coercion** | `valico` (the closest existing match — but **load-bearing abandoned**, see below) | JSON Schema draft-7 + coerce pass before validate | aging; no first-class context, no structured preprocessor pipeline, no Lua hook, no 2020-12; **maintainer inactive since 2023** | +| Approach | Crates | What it covers | Gap from this TODO | +| ----------------------------------------- | --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------- | +| **Schema-as-Rust-type** | `validator`, `garde`, `nutype`, `serde` + custom impls | known-at-compile-time schemas, derived from struct definitions | wrong shape — schema is code, not data | +| **Schema-as-data, validation-only** | `jsonschema` | JSON Schema 2020-12 validation, custom keywords supported, no first-class context, no coercion, no preprocessor pipeline | the foundation we'd build _on top of_ | +| **Schema-as-data, validation + coercion** | `valico` (the closest existing match — but **load-bearing abandoned**, see below) | JSON Schema draft-7 + coerce pass before validate | aging; no first-class context, no structured preprocessor pipeline, no Lua hook, no 2020-12; **maintainer inactive since 2023** | -The healthy foundation is `jsonschema` (Stranger6667 / Dmitry Dygalo). It's actively maintained, supports custom keywords, and tracks JSON Schema 2020-12. The proposed extension crates would depend on it, not fork it. +The healthy foundation is `jsonschema` (Stranger6667 / Dmitry Dygalo). It's +actively maintained, supports custom keywords, and tracks JSON Schema 2020-12. +The proposed extension crates would depend on it, not fork it. ### `valico` is load-bearing abandoned Audit performed 2026-05-06 (https://github.com/s-panferov/valico): -| Signal | Value | -|---|---| -| Last commit on `master` | 2023-05-13 (the v4.0.0 release commit) | -| Last release on crates.io | 4.0.0 — 2023-05-13 | -| Total downloads | 20.9M | -| Recent downloads (90d) | 1.85M — heavily used in production | -| Open issues | 15, all 5 most recent (June 2023 – June 2024) with 0 maintainer comments | -| Open PRs | 2, unmerged | -| Notable active forks | none | -| Archived flag | no | - -This is "load-bearing abandoned" — many production crates depend on it, but the maintainer has been silent for ~3 years. PRs would not be merged. Issues would not be answered. Useful as a reference for ideas (the coerce-then-validate ordering, error reporting patterns), not as a foundation to extend. - -`pydantic-core` (Pydantic V2's Rust core) is **not available** to Rust developers. Confirmed via Samuel Colvin in [pydantic#4851](https://github.com/pydantic/pydantic/discussions/4851): tightly coupled to pyo3, no near-term plans to decouple, and Colvin himself notes a pure-Rust version "would likely underperform compared to Serde". So we cannot piggyback on it. +| Signal | Value | +| ------------------------- | ------------------------------------------------------------------------ | +| Last commit on `master` | 2023-05-13 (the v4.0.0 release commit) | +| Last release on crates.io | 4.0.0 — 2023-05-13 | +| Total downloads | 20.9M | +| Recent downloads (90d) | 1.85M — heavily used in production | +| Open issues | 15, all 5 most recent (June 2023 – June 2024) with 0 maintainer comments | +| Open PRs | 2, unmerged | +| Notable active forks | none | +| Archived flag | no | + +This is "load-bearing abandoned" — many production crates depend on it, but the +maintainer has been silent for ~3 years. PRs would not be merged. Issues would +not be answered. Useful as a reference for ideas (the coerce-then-validate +ordering, error reporting patterns), not as a foundation to extend. + +`pydantic-core` (Pydantic V2's Rust core) is **not available** to Rust +developers. Confirmed via Samuel Colvin in +[pydantic#4851](https://github.com/pydantic/pydantic/discussions/4851): tightly +coupled to pyo3, no near-term plans to decouple, and Colvin himself notes a +pure-Rust version "would likely underperform compared to Serde". So we cannot +piggyback on it. ## The Rust audience and the honest selling point -The Rust audience for runtime-defined validation is genuinely narrower than the Python audience for Pydantic. Rust has compile-time types, so the Pydantic motivation ("runtime type safety in a dynamically-typed language") doesn't apply. Runtime-schema validation is an opt-in concern in Rust — only relevant when the schema is itself runtime data: +The Rust audience for runtime-defined validation is genuinely narrower than the +Python audience for Pydantic. Rust has compile-time types, so the Pydantic +motivation ("runtime type safety in a dynamically-typed language") doesn't +apply. Runtime-schema validation is an opt-in concern in Rust — only relevant +when the schema is itself runtime data: - Multi-tenant SaaS where each tenant has a different schema. - Plugin systems where modules ship their own validation. @@ -67,21 +105,27 @@ The Rust audience for runtime-defined validation is genuinely narrower than the - Document validators (mdvs's exact niche). - OpenAPI / JSON Schema runtime tooling (already a real `jsonschema` audience). -This is a **small but real** audience. The honest selling point isn't ergonomics — Rust devs accept verbose code in exchange for predictability and don't reach for bundles to save boilerplate. The selling point per crate is: +This is a **small but real** audience. The honest selling point isn't ergonomics +— Rust devs accept verbose code in exchange for predictability and don't reach +for bundles to save boilerplate. The selling point per crate is: -- *"Your schema needs context-conditional rules — env, tenant, filepath — and you don't want to write a custom keyword for every variant."* -- *"You're validating user input where strict typing is too rigid; you want coercion declared in the schema, not in your Rust code."* -- *"You want declarative pre-validation transforms without baking them into deserialization."* +- _"Your schema needs context-conditional rules — env, tenant, filepath — and + you don't want to write a custom keyword for every variant."_ +- _"You're validating user input where strict typing is too rigid; you want + coercion declared in the schema, not in your Rust code."_ +- _"You want declarative pre-validation transforms without baking them into + deserialization."_ -Each crate, alone, fills a specific gap a Rust dev hits when reaching for `jsonschema`. None of them is a framework. None forces adoption of the others. +Each crate, alone, fills a specific gap a Rust dev hits when reaching for +`jsonschema`. None of them is a framework. None forces adoption of the others. ## The three-crate sketch -| Crate | What it does | Approximate size | Audience | -|---|---|---|---| -| **``** | Extension keyword(s) for `jsonschema` that read from a runtime `Context: HashMap`. Implements the partition + per-context-overlay pattern designed in TODO-0149. Ships built-in keywords (`x-required-when`, `x-allowed-when`) plus a registry for user-defined ones. | ~500 LOC | OpenAPI tooling, multi-tenant configs, mdvs | -| **``** | Standalone coerce-before-validate logic — declarative per-field rules (`"1"` → `1`, `"true"` → `true`, `"2026-05-06"` → `Date`). Schema-driven; emits structured errors on failed coercion. | ~400 LOC | web servers, REST APIs, config validators | -| **``** | Three-stage pipeline runner: rename keys → transform values → transform document. Each stage is a closed enum of built-ins (lowercase, trim, normalize-unicode, etc.) plus a Lua escape hatch (TODO-0144) for user-defined transforms. | ~600 LOC + Lua bridge | data ingestion, document linters, anything reading messy input | +| Crate | What it does | Approximate size | Audience | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------- | -------------------------------------------------------------- | +| **``** | Extension keyword(s) for `jsonschema` that read from a runtime `Context: HashMap`. Implements the partition + per-context-overlay pattern designed in TODO-0149. Ships built-in keywords (`x-required-when`, `x-allowed-when`) plus a registry for user-defined ones. | ~500 LOC | OpenAPI tooling, multi-tenant configs, mdvs | +| **``** | Standalone coerce-before-validate logic — declarative per-field rules (`"1"` → `1`, `"true"` → `true`, `"2026-05-06"` → `Date`). Schema-driven; emits structured errors on failed coercion. | ~400 LOC | web servers, REST APIs, config validators | +| **``** | Three-stage pipeline runner: rename keys → transform values → transform document. Each stage is a closed enum of built-ins (lowercase, trim, normalize-unicode, etc.) plus a Lua escape hatch (TODO-0144) for user-defined transforms. | ~600 LOC + Lua bridge | data ingestion, document linters, anything reading messy input | Each crate: @@ -107,55 +151,101 @@ crates/ └── cli layer ``` -mdvs's special parts shrink dramatically. The inference algorithm becomes the centerpiece. Validation becomes a composition of library calls. +mdvs's special parts shrink dramatically. The inference algorithm becomes the +centerpiece. Validation becomes a composition of library calls. ## Path chosen (was C in earlier framing) -| Path | Description | Trade-off | -|---|---|---| -| A | TODO-0152 only — internal layering, no extraction | small commitment, clean internals, no general-purpose contribution | -| B | Build the three crates as siblings during Wave B; mdvs consumes them from day one | most ambitious; triples surface area before mdvs has audience | +| Path | Description | Trade-off | +| -------------- | ------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| A | TODO-0152 only — internal layering, no extraction | small commitment, clean internals, no general-purpose contribution | +| B | Build the three crates as siblings during Wave B; mdvs consumes them from day one | most ambitious; triples surface area before mdvs has audience | | **C (chosen)** | **Internal layering now (TODO-0152); extract crates post-mdvs-1.0 if/when triggered** | safest; validates mdvs's audience first; extraction becomes obvious if/when concrete demand exists | -C: TODO-0152 lands first, the extraction concept lives in this TODO as a captured option, no commitment to build any of the three crates yet. +C: TODO-0152 lands first, the extraction concept lives in this TODO as a +captured option, no commitment to build any of the three crates yet. ## Triggers for promoting this from "deferred" to "active" Any one of: -1. **External demand**: an issue, blog comment, or PR asking "can I use mdvs's context-keyword logic / coercion / preprocessor pipeline as a library?" — concrete evidence that someone wants one of the three crates separately. -2. **Second consumer in our own work**: building a second tool (config validator, document linter, ETL pipeline, etc.) that wants the same components. Two consumers force the API into honesty. -3. **mdvs hits a meaningful audience**: post-1.0, with stars/users/issues. At that point spinning out the crates is a discoverability move, not a speculative one. -4. **The niche is filled by someone else**: a new actively-maintained crate ships with the same shape (any one of the three components) — in which case the corresponding part of this TODO is closed `superseded` and mdvs adopts the external crate instead. We don't need to be the ones building this; we just need the niche to be filled. - -Note: contributing context support upstream to `jsonschema` itself was considered as a path and **rejected** — direction this TODO doesn't pursue. If the work happens, it ships as separate crates owned here, not as upstream PRs. +1. **External demand**: an issue, blog comment, or PR asking "can I use mdvs's + context-keyword logic / coercion / preprocessor pipeline as a library?" — + concrete evidence that someone wants one of the three crates separately. +2. **Second consumer in our own work**: building a second tool (config + validator, document linter, ETL pipeline, etc.) that wants the same + components. Two consumers force the API into honesty. +3. **mdvs hits a meaningful audience**: post-1.0, with stars/users/issues. At + that point spinning out the crates is a discoverability move, not a + speculative one. +4. **The niche is filled by someone else**: a new actively-maintained crate + ships with the same shape (any one of the three components) — in which case + the corresponding part of this TODO is closed `superseded` and mdvs adopts + the external crate instead. We don't need to be the ones building this; we + just need the niche to be filled. + +Note: contributing context support upstream to `jsonschema` itself was +considered as a path and **rejected** — direction this TODO doesn't pursue. If +the work happens, it ships as separate crates owned here, not as upstream PRs. ## Prerequisite if/when the work begins -Before any implementation work, the prep is per-crate, not all-at-once. Each of the three is a separate decision with its own design questions: +Before any implementation work, the prep is per-crate, not all-at-once. Each of +the three is a separate decision with its own design questions: -**For the context-keyword crate**, the load-bearing question is the API surface: what does `Context` look like (typed? string-keyed? hierarchical?), how do consumers register keywords, how do errors compose with `jsonschema`'s native error stream? The partition + per-context-overlay pattern from TODO-0149 is the implementation; the API is what needs design. +**For the context-keyword crate**, the load-bearing question is the API surface: +what does `Context` look like (typed? string-keyed? hierarchical?), how do +consumers register keywords, how do errors compose with `jsonschema`'s native +error stream? The partition + per-context-overlay pattern from TODO-0149 is the +implementation; the API is what needs design. -**For the coerce crate**, the load-bearing question is coercion semantics: opt-in per-field vs. global; explicit rules in the schema vs. inferred from `type`; which coercions are sound (string→number is fine; what about string→date?); how to handle ambiguous cases. Read `valico`'s source for ideas, but don't inherit its design — its draft-7 era choices may not generalize cleanly. +**For the coerce crate**, the load-bearing question is coercion semantics: +opt-in per-field vs. global; explicit rules in the schema vs. inferred from +`type`; which coercions are sound (string→number is fine; what about +string→date?); how to handle ambiguous cases. Read `valico`'s source for ideas, +but don't inherit its design — its draft-7 era choices may not generalize +cleanly. -**For the preprocess crate**, the load-bearing question is the Lua bridge: when is Lua loaded, how is it sandboxed, what's the error story when a Lua script panics, how does it interact with the closed-enum built-ins? TODO-0144 holds the broader Lua scope; this crate would expose a thin, focused subset of it. +**For the preprocess crate**, the load-bearing question is the Lua bridge: when +is Lua loaded, how is it sandboxed, what's the error story when a Lua script +panics, how does it interact with the closed-enum built-ins? TODO-0144 holds the +broader Lua scope; this crate would expose a thin, focused subset of it. -Each prerequisite is ~1 week of design work, producing a written assessment, before any code is committed. Skipping any of them risks shipping APIs we'll regret. +Each prerequisite is ~1 week of design work, producing a written assessment, +before any code is committed. Skipping any of them risks shipping APIs we'll +regret. ## Out of scope -- Implementation of any of the three crates. This TODO documents the concept; each crate's actual implementation is a separate multi-month project, gated on the triggers above. -- mdvs's adoption of the crates. Would be a separate migration TODO once they exist. -- Naming. The "Pydantic for Rust" framing is **rejected** — wrong audience signal. Names should be small, factual, descriptive of the gap they fill (e.g., `jsonschema-context`, `json-coerce`, `json-preprocess`). Settle naming only when work begins on a specific crate. -- Multi-language bindings (Python via pyo3, JS via wasm-bindgen, etc.). Audited informally during TODO-0153 design discussion and rejected — diluted focus, narrow per-language audience, no compelling differentiator against natives. Could revisit only if a specific cross-language document-toolchain pitch becomes concrete. +- Implementation of any of the three crates. This TODO documents the concept; + each crate's actual implementation is a separate multi-month project, gated on + the triggers above. +- mdvs's adoption of the crates. Would be a separate migration TODO once they + exist. +- Naming. The "Pydantic for Rust" framing is **rejected** — wrong audience + signal. Names should be small, factual, descriptive of the gap they fill + (e.g., `jsonschema-context`, `json-coerce`, `json-preprocess`). Settle naming + only when work begins on a specific crate. +- Multi-language bindings (Python via pyo3, JS via wasm-bindgen, etc.). Audited + informally during TODO-0153 design discussion and rejected — diluted focus, + narrow per-language audience, no compelling differentiator against natives. + Could revisit only if a specific cross-language document-toolchain pitch + becomes concrete. ## Acceptance criteria -This TODO has no acceptance criteria — it's a captured design idea, not a planned piece of work. It's "complete" when one of the triggers fires for one of the three crates, and a successor TODO is created for that crate (which would then have its own acceptance criteria). +This TODO has no acceptance criteria — it's a captured design idea, not a +planned piece of work. It's "complete" when one of the triggers fires for one of +the three crates, and a successor TODO is created for that crate (which would +then have its own acceptance criteria). ## Related -- TODO-0149 — the JSON Schema refoundation; the validation pipeline this TODO would extract pieces of -- TODO-0152 — the layered internal structure; the natural precursor to extraction -- TODO-0144 — Lua scripting; the preprocess crate would expose Lua as one of its built-in mechanisms -- TODO-0150 — strict mode; would be implemented at the coerce-crate layer once it exists, not in mdvs +- TODO-0149 — the JSON Schema refoundation; the validation pipeline this TODO + would extract pieces of +- TODO-0152 — the layered internal structure; the natural precursor to + extraction +- TODO-0144 — Lua scripting; the preprocess crate would expose Lua as one of its + built-in mechanisms +- TODO-0150 — strict mode; would be implemented at the coerce-crate layer once + it exists, not in mdvs diff --git a/docs/spec/todos/TODO-0154.md b/docs/spec/todos/TODO-0154.md index 56f3dd4..f9f7f77 100644 --- a/docs/spec/todos/TODO-0154.md +++ b/docs/spec/todos/TODO-0154.md @@ -12,22 +12,38 @@ blocks: [] ## Summary -TODO-0149 step 13 ("per-file overlay synthesis") replaces the Rust-side `globset` path-scoping in `cmd/check.rs` with a per-file synthesized JSON Schema overlay, validated via `jsonschema::Validator`. The naive implementation compiles one `Validator` per file. For most files in a project, the overlay is **identical** — files in the same directory subtree share the same set of matching `x-mdvs.allowed`/`required` globs. - -This TODO captures the optimization: compute a signature per file ("which fields' globs matched this path?"), cache compiled overlays by signature, reuse the same `Validator` across files that share an overlay. - -**Step 13 ships with this optimization in place.** The naive (no-cache) version exists in design discussion but was rejected before implementation because the per-file compile cost (~10–100µs × N files) accumulates noticeably at project scales above a few thousand files. Better to ship the optimized form than ship the slow form and patch later. +TODO-0149 step 13 ("per-file overlay synthesis") replaces the Rust-side +`globset` path-scoping in `cmd/check.rs` with a per-file synthesized JSON Schema +overlay, validated via `jsonschema::Validator`. The naive implementation +compiles one `Validator` per file. For most files in a project, the overlay is +**identical** — files in the same directory subtree share the same set of +matching `x-mdvs.allowed`/`required` globs. + +This TODO captures the optimization: compute a signature per file ("which +fields' globs matched this path?"), cache compiled overlays by signature, reuse +the same `Validator` across files that share an overlay. + +**Step 13 ships with this optimization in place.** The naive (no-cache) version +exists in design discussion but was rejected before implementation because the +per-file compile cost (~10–100µs × N files) accumulates noticeably at project +scales above a few thousand files. Better to ship the optimized form than ship +the slow form and patch later. ## Why this exists as a separate TODO Conceptually independent from step 13: -- Step 13 changes the **mechanism** of path-scoping (globset → JSON Schema overlay). -- This TODO changes the **performance characteristics** of that mechanism (per-file → per-signature). +- Step 13 changes the **mechanism** of path-scoping (globset → JSON Schema + overlay). +- This TODO changes the **performance characteristics** of that mechanism + (per-file → per-signature). -Bundling them in one commit risks conflating "did we get the mechanism right?" with "did we get the cache right?" Keeping them separate makes review easier and lets either piece be reverted independently if a regression surfaces. +Bundling them in one commit risks conflating "did we get the mechanism right?" +with "did we get the cache right?" Keeping them separate makes review easier and +lets either piece be reverted independently if a regression surfaces. -In practice they ship together (step 13 is gated on this TODO landing), but the separation is structural. +In practice they ship together (step 13 is gated on this TODO landing), but the +separation is structural. ## Approach @@ -38,9 +54,11 @@ For each field `f` in the canonical schema, the signature records: - Did `f.allowed` match the file's path? (1 bit) - Did `f.required` match the file's path? (1 bit) -For N fields, that's 2N bits. Encoded as a packed `Vec` or a `Vec` keyed in a `HashMap`. +For N fields, that's 2N bits. Encoded as a packed `Vec` or a `Vec` +keyed in a `HashMap`. -Two files with identical signatures produce identical overlays — same `properties` map, same `required` array. +Two files with identical signatures produce identical overlays — same +`properties` map, same `required` array. ### Cache shape @@ -70,33 +88,48 @@ Build once at top of `cmd/check.rs::validate`, hand to `check_field_values`. ### Expected cache hit rate -For `example_kb` (43 files, 39 fields, ~4 top-level directories): ~5–10 distinct signatures expected (one per directory tree node where globs change). +For `example_kb` (43 files, 39 fields, ~4 top-level directories): ~5–10 distinct +signatures expected (one per directory tree node where globs change). -For larger projects with deeper directory structure but consistent glob patterns (most cases), the ratio stays favorable: number of distinct signatures grows with the number of distinct glob-applicability regions, not with the number of files. +For larger projects with deeper directory structure but consistent glob patterns +(most cases), the ratio stays favorable: number of distinct signatures grows +with the number of distinct glob-applicability regions, not with the number of +files. ## Files -- `crates/mdvs/src/schema/overlay.rs` (created in step 13): add `OverlayCache` struct and `validator_for(file_path)` method. -- `crates/mdvs/src/cmd/check.rs::validate`: build cache once, pass mutable ref into `check_field_values`. +- `crates/mdvs/src/schema/overlay.rs` (created in step 13): add `OverlayCache` + struct and `validator_for(file_path)` method. +- `crates/mdvs/src/cmd/check.rs::validate`: build cache once, pass mutable ref + into `check_field_values`. ## Tests -- **Cache hit unit test**: two files with same signature → same Validator (compare pointer identity or compile-count counter). -- **Cache miss unit test**: files producing different signatures → separate validators. -- **Behavior parity test**: same fixture exercising the cached and uncached paths produces identical violations. -- **Stress test (optional)**: large synthetic fixture (1000+ files across 10 directories) — confirm only ~10 validators compiled. +- **Cache hit unit test**: two files with same signature → same Validator + (compare pointer identity or compile-count counter). +- **Cache miss unit test**: files producing different signatures → separate + validators. +- **Behavior parity test**: same fixture exercising the cached and uncached + paths produces identical violations. +- **Stress test (optional)**: large synthetic fixture (1000+ files across 10 + directories) — confirm only ~10 validators compiled. ## Verification -- `cargo bench` or ad-hoc timing on `example_kb` and a larger fixture: confirm sub-millisecond path-scoping phase regardless of file count. +- `cargo bench` or ad-hoc timing on `example_kb` and a larger fixture: confirm + sub-millisecond path-scoping phase regardless of file count. - `mdvs check example_kb`: parity preserved. ## Out of scope -- Caching across mdvs invocations (would require disk persistence; not worth it). -- Caching against `validate_mdvs_schema` results (the gate runs once per command, no caching needed). -- Pre-computing signatures during scan (the file-system walk already knows paths; signature computation could be inlined there, but that mixes concerns). +- Caching across mdvs invocations (would require disk persistence; not worth + it). +- Caching against `validate_mdvs_schema` results (the gate runs once per + command, no caching needed). +- Pre-computing signatures during scan (the file-system walk already knows + paths; signature computation could be inlined there, but that mixes concerns). ## Related -- TODO-0149: step 13 introduces the overlay mechanism this TODO optimizes. Step 13 doesn't ship without this in place. +- TODO-0149: step 13 introduces the overlay mechanism this TODO optimizes. Step + 13 doesn't ship without this in place. diff --git a/docs/spec/todos/TODO-0155.md b/docs/spec/todos/TODO-0155.md index de769d2..6d90deb 100644 --- a/docs/spec/todos/TODO-0155.md +++ b/docs/spec/todos/TODO-0155.md @@ -16,23 +16,25 @@ related: > **Closeout (2026-05-13).** All four in-repo batches shipped on branch > `feat/todo-0155-syntax`. On-disk form is now function-style strings -> (`type = "Array(String)"`); `Array(Object{...})` rejected at parse, -> inference, and config-load (invariant 9). example_kb migrated to -> parallel scalar arrays. Refractions re-baselined (vault left dirty -> for the user to commit). Three commits ahead of `feat/todo-0149`: -> `d51c7a2` (parser + serde + example_kb), `5525c07` (inference + invariant 9), -> `1918522` (docs + mdBook sweep). Tests: 699 passing. +> (`type = "Array(String)"`); `Array(Object{...})` rejected at parse, inference, +> and config-load (invariant 9). example_kb migrated to parallel scalar arrays. +> Refractions re-baselined (vault left dirty for the user to commit). Three +> commits ahead of `feat/todo-0149`: `d51c7a2` (parser + serde + example_kb), +> `5525c07` (inference + invariant 9), `1918522` (docs + mdBook sweep). Tests: +> 699 passing. ## Summary -mdvs's TOML type representation has two co-existing forms that diverge between the file-on-disk and the user-visible output: +mdvs's TOML type representation has two co-existing forms that diverge between +the file-on-disk and the user-visible output: -| Context | Form | -|---|---| +| Context | Form | +| -------------------------------- | ----------------------------- | | `mdvs.toml` (serde inline-table) | `type = { array = "String" }` | -| Console output (Display) | `Array(String)` | +| Console output (Display) | `Array(String)` | -This TODO collapses the dual form into one. **Adopt function-style on disk** — the same vocabulary CLI Display already emits. +This TODO collapses the dual form into one. **Adopt function-style on disk** — +the same vocabulary CLI Display already emits. ```toml [[fields.field]] @@ -40,7 +42,9 @@ name = "tags" type = "Array(String)" ``` -The "loses TOML structured-typing benefits" cost is paid by writing a small hand-rolled parser in `FieldTypeSerde::deserialize`. The grammar is bounded and small. +The "loses TOML structured-typing benefits" cost is paid by writing a small +hand-rolled parser in `FieldTypeSerde::deserialize`. The grammar is bounded and +small. ## Why function-style @@ -51,11 +55,20 @@ The "loses TOML structured-typing benefits" cost is paid by writing a small hand ## Out of scope (moved to TODO-0156) -The original 155 was **expanded** to include reusable definitions via `$defs` / `$ref` (an `[definitions]` section + `@Name` reference syntax). That expansion has been split back out into **[TODO-0156](TODO-0156.md)** (low priority) because: - -- `[definitions]` + `Array(@SensorReading)` inevitably forces re-introducing inline `Object{name: String, ...}` definitions on disk — directly contradicting Wave C's "no inline structured types" philosophy (TODO-0097). -- The two changes have very different priorities: vocabulary unification is high (cognitive load is real today), reusable Array-of-Object representation is low (no users are asking for it). -- Splitting also lets this TODO commit to **rejecting** `Array(Object{...})` entirely as part of the new strict grammar — a forcing function that keeps the surface clean. +The original 155 was **expanded** to include reusable definitions via `$defs` / +`$ref` (an `[definitions]` section + `@Name` reference syntax). That expansion +has been split back out into **[TODO-0156](TODO-0156.md)** (low priority) +because: + +- `[definitions]` + `Array(@SensorReading)` inevitably forces re-introducing + inline `Object{name: String, ...}` definitions on disk — directly + contradicting Wave C's "no inline structured types" philosophy (TODO-0097). +- The two changes have very different priorities: vocabulary unification is high + (cognitive load is real today), reusable Array-of-Object representation is low + (no users are asking for it). +- Splitting also lets this TODO commit to **rejecting** `Array(Object{...})` + entirely as part of the new strict grammar — a forcing function that keeps the + surface clean. ## Grammar @@ -65,32 +78,54 @@ Scalar := "String" | "Integer" | "Float" | "Boolean" Array := "Array(" Scalar ")" ``` -That's the entire allowed grammar for the `type` field in mdvs.toml. Explicitly rejected: +That's the entire allowed grammar for the `type` field in mdvs.toml. Explicitly +rejected: - `Array(Array(...))` — already disallowed today; stays disallowed. -- `Array(Object{...})` — **newly rejected**. The escape hatch is parallel scalar arrays (`measurement_timestamps: Array(String)`, `measurement_values: Array(Float)`). -- `Object{...}` as a parseable type — top-level Object is already rejected by config-load invariant 6 (Wave C). After this TODO, there is no valid place to write `Object{...}` in `mdvs.toml` at all. +- `Array(Object{...})` — **newly rejected**. The escape hatch is parallel scalar + arrays (`measurement_timestamps: Array(String)`, + `measurement_values: Array(Float)`). +- `Object{...}` as a parseable type — top-level Object is already rejected by + config-load invariant 6 (Wave C). After this TODO, there is no valid place to + write `Object{...}` in `mdvs.toml` at all. - `@RefName` — deferred to TODO-0156. -`FieldType::Object` continues to exist internally — it's the synthetic transposition target for Wave C's dotted-name leaves (`storage.rs::transpose_to_storage_type` builds Object trees from flat-toml). The serde surface never produces or consumes it. +`FieldType::Object` continues to exist internally — it's the synthetic +transposition target for Wave C's dotted-name leaves +(`storage.rs::transpose_to_storage_type` builds Object trees from flat-toml). +The serde surface never produces or consumes it. ## Display stays a superset of parse -CLI output still renders `Object{...}` and `Array(Object{...})` in violation messages and other diagnostics. Display describes shapes (including unsupported ones, for error reporting); parse is the gate that enforces what's legal. This asymmetry is deliberate. +CLI output still renders `Object{...}` and `Array(Object{...})` in violation +messages and other diagnostics. Display describes shapes (including unsupported +ones, for error reporting); parse is the gate that enforces what's legal. This +asymmetry is deliberate. ## Inference behavior on Array-of-mappings -If `mdvs init` encounters frontmatter containing `field: [{k: v, ...}, ...]` (an array of objects): +If `mdvs init` encounters frontmatter containing `field: [{k: v, ...}, ...]` (an +array of objects): -- **`init`** — the field is rejected from `[[fields.field]]` synthesis and surfaced via a warning. It does NOT end up in `[fields].ignore` automatically (the user might want it). Print a clear message: `field "X" contains Array(Object) which isn't representable; consider flattening into parallel arrays`. -- **`check`** — same situation against an existing config: produce a document-level `FrontmatterUnrepresentable` violation (same machinery as Wave B step 8 for YAML→JSON silent drops). -- **`update`** — same as init: surface a warning, leave the field out of fresh synthesis. +- **`init`** — the field is rejected from `[[fields.field]]` synthesis and + surfaced via a warning. It does NOT end up in `[fields].ignore` automatically + (the user might want it). Print a clear message: + `field "X" contains Array(Object) which isn't representable; consider flattening into parallel arrays`. +- **`check`** — same situation against an existing config: produce a + document-level `FrontmatterUnrepresentable` violation (same machinery as Wave + B step 8 for YAML→JSON silent drops). +- **`update`** — same as init: surface a warning, leave the field out of fresh + synthesis. -This is consistent with the Wave C invariant 6 messaging style: clear rejection at the right layer, not silent acceptance + downstream confusion. +This is consistent with the Wave C invariant 6 messaging style: clear rejection +at the right layer, not silent acceptance + downstream confusion. ## Parser implementation -Hand-written recursive descent in a new `FieldTypeSerde::parse(&str) -> Result`. No parser-combinator dependency — the grammar is trivial and TOML users need precise error messages with column offsets. +Hand-written recursive descent in a new +`FieldTypeSerde::parse(&str) -> Result`. No parser-combinator +dependency — the grammar is trivial and TOML users need precise error messages +with column offsets. ```rust impl<'de> Deserialize<'de> for FieldTypeSerde { @@ -101,44 +136,65 @@ impl<'de> Deserialize<'de> for FieldTypeSerde { } ``` -Hard cutover — drop the inline-table deserialize arm entirely (per "we have no users" rule, dual-form maintenance cost isn't worth it). example_kb regenerates in the same commit. Refractions vault regenerates by running `mdvs init --force` once (already done as the deep-test re-baseline). +Hard cutover — drop the inline-table deserialize arm entirely (per "we have no +users" rule, dual-form maintenance cost isn't worth it). example_kb regenerates +in the same commit. Refractions vault regenerates by running `mdvs init --force` +once (already done as the deep-test re-baseline). -Roundtrip property test: `parse(Display(ft)) == Ok(ft)` for every supported `FieldTypeSerde` value. +Roundtrip property test: `parse(Display(ft)) == Ok(ft)` for every supported +`FieldTypeSerde` value. ## Files touched ### Code -- **`crates/mdvs/src/schema/shared.rs`** — add `FieldTypeSerde::parse` + `ParseError` type. Drop the inline-table deserialize arm. Display impl already function-style, no change. -- **`crates/mdvs/src/schema/config.rs`** — new validate invariant: reject `FieldType::Object` and `FieldType::Array(FieldType::Object(_))` at config load with a clear error pointing at the offending field. (Invariant 8 already covers leaf-vs-parent shape conflicts; new invariant 9 covers the Array(Object) rejection.) -- **`crates/mdvs/src/discover/infer/types.rs`** — when widening lands on `Array(Object)`, emit a warning to stderr and skip the field from inference output. Document via `feedback_array_of_object_unrepresentable.md` in memory. -- **`crates/mdvs/src/cmd/check.rs`** — wire the `Array(Object)` case into `FrontmatterUnrepresentable` violations for the validation path. +- **`crates/mdvs/src/schema/shared.rs`** — add `FieldTypeSerde::parse` + + `ParseError` type. Drop the inline-table deserialize arm. Display impl already + function-style, no change. +- **`crates/mdvs/src/schema/config.rs`** — new validate invariant: reject + `FieldType::Object` and `FieldType::Array(FieldType::Object(_))` at config + load with a clear error pointing at the offending field. (Invariant 8 already + covers leaf-vs-parent shape conflicts; new invariant 9 covers the + Array(Object) rejection.) +- **`crates/mdvs/src/discover/infer/types.rs`** — when widening lands on + `Array(Object)`, emit a warning to stderr and skip the field from inference + output. Document via `feedback_array_of_object_unrepresentable.md` in memory. +- **`crates/mdvs/src/cmd/check.rs`** — wire the `Array(Object)` case into + `FrontmatterUnrepresentable` violations for the validation path. ### Tests -- `parse_and_display_round_trip` (property-style table) covering every `Scalar` and `Array(Scalar)` combination. +- `parse_and_display_round_trip` (property-style table) covering every `Scalar` + and `Array(Scalar)` combination. - `parse_rejects_array_of_object`. - `parse_rejects_object_top_level`. - `parse_rejects_array_of_array`. - `parse_error_points_to_column` — error messages include a column offset. -- `config_load_rejects_array_of_object` — config-level rejection at invariant check. -- `infer_skips_array_of_object_field` — inference behavior on real Array-of-mappings input. +- `config_load_rejects_array_of_object` — config-level rejection at invariant + check. +- `infer_skips_array_of_object_field` — inference behavior on real + Array-of-mappings input. ### Specs -- `docs/spec/architecture.md` — type system section: function-style is the canonical form on disk. Brief grammar table. +- `docs/spec/architecture.md` — type system section: function-style is the + canonical form on disk. Brief grammar table. - `docs/spec/commands/init.md` — note the Array-of-Object warning behavior. -- `docs/spec/commands/check.md` — note the `FrontmatterUnrepresentable` extension for Array-of-Object. +- `docs/spec/commands/check.md` — note the `FrontmatterUnrepresentable` + extension for Array-of-Object. ### mdBook -- `book/src/concepts/types.md` — replace inline-table examples with function-style; add note about Array(Object) being unsupported. +- `book/src/concepts/types.md` — replace inline-table examples with + function-style; add note about Array(Object) being unsupported. - `book/src/configuration.md` — same replacements. - `book/src/concepts/validation.md` — sanity check examples. ### example_kb -- Replace the `measurements: Array(Object{timestamp, value})` field with two parallel arrays: +- Replace the `measurements: Array(Object{timestamp, value})` field with two + parallel arrays: + ```toml [[fields.field]] name = "measurement_timestamps" @@ -149,21 +205,30 @@ Roundtrip property test: `parse(Display(ft)) == Ok(ft)` for every supported `Fie name = "measurement_values" type = "Array(Float)" allowed = ["projects/alpha/notes/**"] + ``` + - Update `experiment-2.md` frontmatter to use the new shape: + ```yaml measurement_timestamps: ["14:02:11", "14:03:00"] measurement_values: [0.612, 0.598] + ``` -- This demonstrates the recommended migration recipe for any users hitting the rejection. + +- This demonstrates the recommended migration recipe for any users hitting the + rejection. ### Refractions -- Regenerate `mdvs.toml` with `init --force` after this lands; assume no Array(Object) fields based on the deep-test (only mixed-type widening surfaced — no nested mappings observed). Verify before commit. +- Regenerate `mdvs.toml` with `init --force` after this lands; assume no + Array(Object) fields based on the deep-test (only mixed-type widening surfaced + — no nested mappings observed). Verify before commit. ## Execution plan -Five batches. Each batch leaves the tree green. Estimated 3 commits in mdvs + 1 in Refractions, ~600–800 lines of net change. +Five batches. Each batch leaves the tree green. Estimated 3 commits in mdvs + 1 +in Refractions, ~600–800 lines of net change. ### Batch 1 — Parser + serde swap (the breaking change) @@ -171,14 +236,25 @@ Goal: function-style strings parse and round-trip; inline-table form is gone. In `crates/mdvs/src/schema/shared.rs`: -- Add `ParseError` struct (`message: String`, `column: usize`) with `Display` + `Error` impls. -- Add `FieldTypeSerde::parse(&str) -> Result` — hand-written recursive descent. Grammar: `Scalar | Array(Scalar)`. Anything else (`Object{...}`, `Array(Object{...})`, `Array(Array(...))`, `@Ref`) errors at parse with a column offset and a hint message; the Array-of-Object case points users at TODO-0156. -- Replace `FieldTypeSerde::deserialize` body: deserialize a `String`, call `parse`, propagate via `D::Error::custom`. Drop the inline-table arm entirely. -- Replace `FieldTypeSerde::serialize` body: emit `self.to_string()` (Display is already function-style). Drop whatever currently produces the inline-table form. - -Drop the `inline_field_types` post-processor in `schema/config.rs` for the `type` key (becomes redundant once `Serialize` produces a string directly). `reorder_field_keys` stays. +- Add `ParseError` struct (`message: String`, `column: usize`) with `Display` + + `Error` impls. +- Add `FieldTypeSerde::parse(&str) -> Result` — hand-written + recursive descent. Grammar: `Scalar | Array(Scalar)`. Anything else + (`Object{...}`, `Array(Object{...})`, `Array(Array(...))`, `@Ref`) errors at + parse with a column offset and a hint message; the Array-of-Object case points + users at TODO-0156. +- Replace `FieldTypeSerde::deserialize` body: deserialize a `String`, call + `parse`, propagate via `D::Error::custom`. Drop the inline-table arm entirely. +- Replace `FieldTypeSerde::serialize` body: emit `self.to_string()` (Display is + already function-style). Drop whatever currently produces the inline-table + form. + +Drop the `inline_field_types` post-processor in `schema/config.rs` for the +`type` key (becomes redundant once `Serialize` produces a string directly). +`reorder_field_keys` stays. Tests in the same module: + - `parse_scalar_all_four` - `parse_array_of_scalar_all_four` - `parse_rejects_array_of_object_with_column_offset` @@ -187,14 +263,20 @@ Tests in the same module: - `parse_rejects_at_ref` - `roundtrip_parse_display_property_table` - `parse_whitespace_tolerant` (`" Array( String ) "` parses) -- Display asymmetry: `display_renders_unsupported_object_for_diagnostics` (Display still produces `Object{...}` and `Array(Object{...})` even though parse rejects them — pin this). +- Display asymmetry: `display_renders_unsupported_object_for_diagnostics` + (Display still produces `Object{...}` and `Array(Object{...})` even though + parse rejects them — pin this). ### Batch 2 — Fixture migration (coupled with Batch 1) -Lands in the same commit as Batch 1. The test suite is red between Batches 1 and 2. +Lands in the same commit as Batch 1. The test suite is red between Batches 1 +and 2. + +- `grep -rn 'type = {' crates/mdvs/src/ crates/mdvs/tests/` — convert each + occurrence to the function-style string. +- `example_kb/mdvs.toml` — convert `action_items`, `attendees`, `tags` to + `type = "Array(String)"`. Split `measurements` into two fields: -- `grep -rn 'type = {' crates/mdvs/src/ crates/mdvs/tests/` — convert each occurrence to the function-style string. -- `example_kb/mdvs.toml` — convert `action_items`, `attendees`, `tags` to `type = "Array(String)"`. Split `measurements` into two fields: ```toml [[fields.field]] name = "measurement_timestamps" @@ -205,61 +287,107 @@ Lands in the same commit as Batch 1. The test suite is red between Batches 1 and name = "measurement_values" type = "Array(Float)" allowed = ["projects/alpha/notes/**"] + ``` + - `example_kb/projects/alpha/notes/experiment-2.md` — frontmatter reshape: + ```yaml measurement_timestamps: ["14:02:11", "14:03:00"] measurement_values: [0.612, 0.598] + ``` -- Top-of-file warning comment in `example_kb/mdvs.toml`: extend with a note about the parallel-arrays pattern being the recommended migration when Array(Object) is needed. + +- Top-of-file warning comment in `example_kb/mdvs.toml`: extend with a note + about the parallel-arrays pattern being the recommended migration when + Array(Object) is needed. Verify: + - `cargo clippy --all-targets -p mdvs` + `cargo fmt` clean. - `cargo test -p mdvs` green. - `cargo run -p mdvs -- check example_kb` zero violations. -- `cargo run -p mdvs -- init --force example_kb` produces a mdvs.toml byte-equivalent to the hand-shaped form (modulo manual constraint sections, which init drops as always). +- `cargo run -p mdvs -- init --force example_kb` produces a mdvs.toml + byte-equivalent to the hand-shaped form (modulo manual constraint sections, + which init drops as always). ### Batch 3 — Invariant 9 + inference skip-with-warning -**Correction to earlier planning:** dropped the `FrontmatterUnrepresentable` angle for `check`. With strict types, Array-of-mappings in a field declared `Array(String)` already produces a `WrongType` violation via the existing `jsonschema` path. No new violation kind needed. Only inference needs new behavior. +**Correction to earlier planning:** dropped the `FrontmatterUnrepresentable` +angle for `check`. With strict types, Array-of-mappings in a field declared +`Array(String)` already produces a `WrongType` violation via the existing +`jsonschema` path. No new violation kind needed. Only inference needs new +behavior. Code: -- **`crates/mdvs/src/schema/config.rs::validate`** — add invariant 9: reject `FieldType::Object(_)` and `FieldType::Array(Box::new(FieldType::Object(_)))` anywhere in any field's type. Defense-in-depth alongside parser rejection (catches `--from-jsonschema` paths that construct `FieldType` directly without going through `parse`). -- **`crates/mdvs/src/discover/infer/types.rs`** — when the final widened type for a field is `Array(Object(_))` or `Object(_)`, drop the field from the inference output. Wave C already flattens top-level Object, but the safety net matters for edge cases. Collect dropped-field names + their inferred shape + first observed file path into a `dropped_array_of_object: Vec` reported back from the inference pass. -- **`crates/mdvs/src/cmd/init.rs`** + **`crates/mdvs/src/cmd/update.rs`** — when inference returns dropped fields, print to stderr: +- **`crates/mdvs/src/schema/config.rs::validate`** — add invariant 9: reject + `FieldType::Object(_)` and `FieldType::Array(Box::new(FieldType::Object(_)))` + anywhere in any field's type. Defense-in-depth alongside parser rejection + (catches `--from-jsonschema` paths that construct `FieldType` directly without + going through `parse`). +- **`crates/mdvs/src/discover/infer/types.rs`** — when the final widened type + for a field is `Array(Object(_))` or `Object(_)`, drop the field from the + inference output. Wave C already flattens top-level Object, but the safety net + matters for edge cases. Collect dropped-field names + their inferred shape + + first observed file path into a `dropped_array_of_object: Vec` + reported back from the inference pass. +- **`crates/mdvs/src/cmd/init.rs`** + **`crates/mdvs/src/cmd/update.rs`** — when + inference returns dropped fields, print to stderr: + ``` warning: skipped field "measurements" — Array(Object{...}) isn't representable first observed in projects/alpha/notes/experiment-2.md consider splitting into parallel arrays (see book/concepts/types.md) + ``` + Do NOT auto-add to `[fields].ignore`. User keeps full control. Tests: + - `infer_drops_array_of_object_with_warning` (unit test on the inference path). -- `infer_drops_object_top_level_with_warning` (sanity safety net; Wave C should make this impossible from a fresh scan, but the invariant matters). +- `infer_drops_object_top_level_with_warning` (sanity safety net; Wave C should + make this impossible from a fresh scan, but the invariant matters). - `config_validate_rejects_array_of_object` (invariant 9 directly). -- `init_warning_message_format` (CLI integration test against a tmp fixture with Array(Object) frontmatter). +- `init_warning_message_format` (CLI integration test against a tmp fixture with + Array(Object) frontmatter). ### Batch 4 — Spec + mdBook sweep Specs: -- `docs/spec/architecture.md` — type system section: function-style is the canonical disk form; grammar block; note that `Object{...}` and `Array(Object{...})` are not parseable. -- `docs/spec/commands/init.md` + `docs/spec/commands/update.md` — document the dropped-field warning. -- `docs/spec/commands/check.md` — clarify that Array-of-mappings against a scalar Array field fires the existing `WrongType` violation. One-liner cross-reference to TODO-0156 for users wanting first-class Array-of-structured-item support. + +- `docs/spec/architecture.md` — type system section: function-style is the + canonical disk form; grammar block; note that `Object{...}` and + `Array(Object{...})` are not parseable. +- `docs/spec/commands/init.md` + `docs/spec/commands/update.md` — document the + dropped-field warning. +- `docs/spec/commands/check.md` — clarify that Array-of-mappings against a + scalar Array field fires the existing `WrongType` violation. One-liner + cross-reference to TODO-0156 for users wanting first-class + Array-of-structured-item support. mdBook: -- `book/src/concepts/types.md` — replace every inline-table example with function-style; add a new section "Array of structured items isn't supported" with the parallel-arrays migration recipe. -- `book/src/configuration.md` — same example replacements + a callout on the grammar. -- `book/src/concepts/validation.md` — sanity-check examples reflect the new form. + +- `book/src/concepts/types.md` — replace every inline-table example with + function-style; add a new section "Array of structured items isn't supported" + with the parallel-arrays migration recipe. +- `book/src/configuration.md` — same example replacements + a callout on the + grammar. +- `book/src/concepts/validation.md` — sanity-check examples reflect the new + form. ### Batch 5 — Refractions re-baseline -Separate repo, separate commit. Run `mdvs init --force` on the Refractions vault to regenerate `mdvs.toml` in the new form. Verify no `Array(Object)` dropped warnings (expected, based on the deep-test). **Ask before pushing to that repo** — never push to Refractions autonomously. +Separate repo, separate commit. Run `mdvs init --force` on the Refractions vault +to regenerate `mdvs.toml` in the new form. Verify no `Array(Object)` dropped +warnings (expected, based on the deep-test). **Ask before pushing to that repo** +— never push to Refractions autonomously. ### Commit plan -- **Commit 1** (Batches 1 + 2): parser + serde swap + fixture migration + example_kb migration. +- **Commit 1** (Batches 1 + 2): parser + serde swap + fixture migration + + example_kb migration. - **Commit 2** (Batch 3): invariant 9 + inference skip + warning + tests. - **Commit 3** (Batch 4): docs + mdBook. - **Commit 4** (Batch 5): Refractions re-baseline (in the Refractions repo). @@ -269,21 +397,35 @@ Separate repo, separate commit. Run `mdvs init --force` on the Refractions vault 1. `cargo clippy --all-targets -p mdvs` clean. 2. `cargo fmt`. 3. `cargo test -p mdvs` green; ~10 new tests added. -4. `cargo run -p mdvs -- init --force example_kb` writes function-style types; resulting mdvs.toml matches the new hand-shaped form (function-style + parallel arrays). +4. `cargo run -p mdvs -- init --force example_kb` writes function-style types; + resulting mdvs.toml matches the new hand-shaped form (function-style + + parallel arrays). 5. `cargo run -p mdvs -- check example_kb` — zero violations. -6. Hand-write `mdvs.toml` with `type = { array = "String" }` (old form) — config load fails with a clear error pointing at the line. -7. Hand-write `mdvs.toml` with `type = "Array(Object{...})"` — parse error with column offset, clear "Array of Object isn't supported; consider parallel arrays" message. -8. `export-jsonschema` → `init --from-jsonschema` round-trip still works (canonical_to_dsl emits function-style now). +6. Hand-write `mdvs.toml` with `type = { array = "String" }` (old form) — config + load fails with a clear error pointing at the line. +7. Hand-write `mdvs.toml` with `type = "Array(Object{...})"` — parse error with + column offset, clear "Array of Object isn't supported; consider parallel + arrays" message. +8. `export-jsonschema` → `init --from-jsonschema` round-trip still works + (canonical_to_dsl emits function-style now). ## Definition of done - One disk form for types: function-style strings. - Old inline-table form rejected at parse with a clear migration message. -- `Array(Object{...})` rejected at parse AND at config load AND at inference, with consistent messaging. +- `Array(Object{...})` rejected at parse AND at config load AND at inference, + with consistent messaging. - example_kb migrated to parallel arrays for the `measurements` use case. - Specs + mdBook + memory updated. -- TODO-0156 created and linked from this one as the follow-up for the rejected feature. +- TODO-0156 created and linked from this one as the follow-up for the rejected + feature. ## Renamed history -Original TODO-0155 was "Reusable type definitions via $defs / $ref" (low priority). Expanded 2026-05-11 to bundle that with display↔serde unification at high priority. Re-scoped 2026-05-13 back to just the unification work after the user pointed out that `Array(@SensorReading)` reintroduces inline `Object{...}` definitions — a deviation from Wave C's explosion philosophy. The reusable-definitions half moved to [TODO-0156](TODO-0156.md), low priority again. +Original TODO-0155 was "Reusable type definitions via $defs / $ref" (low +priority). Expanded 2026-05-11 to bundle that with display↔serde unification at +high priority. Re-scoped 2026-05-13 back to just the unification work after the +user pointed out that `Array(@SensorReading)` reintroduces inline `Object{...}` +definitions — a deviation from Wave C's explosion philosophy. The +reusable-definitions half moved to [TODO-0156](TODO-0156.md), low priority +again. diff --git a/docs/spec/todos/TODO-0156.md b/docs/spec/todos/TODO-0156.md index 53ab1c9..e292907 100644 --- a/docs/spec/todos/TODO-0156.md +++ b/docs/spec/todos/TODO-0156.md @@ -14,7 +14,10 @@ related: ## Summary -After Wave C (TODO-0097) flattened top-level Objects into dotted-name leaves, and after TODO-0155 rejects `Array(Object{...})` outright, mdvs has **no idiomatic way to express "an array of structured items"** in `mdvs.toml`. The v0 workaround is **parallel arrays**: +After Wave C (TODO-0097) flattened top-level Objects into dotted-name leaves, +and after TODO-0155 rejects `Array(Object{...})` outright, mdvs has **no +idiomatic way to express "an array of structured items"** in `mdvs.toml`. The v0 +workaround is **parallel arrays**: ```toml [[fields.field]] @@ -26,19 +29,26 @@ name = "measurement_values" type = "Array(Float)" ``` -This works (and is what example_kb uses post-155) but it's a workaround, not a representation. Parallel arrays: +This works (and is what example_kb uses post-155) but it's a workaround, not a +representation. Parallel arrays: -- Lose the per-item grouping. There's no schema-level guarantee that `measurement_timestamps[3]` and `measurement_values[3]` belong to the same record. +- Lose the per-item grouping. There's no schema-level guarantee that + `measurement_timestamps[3]` and `measurement_values[3]` belong to the same + record. - Spread one logical field across N TOML field blocks. -- Don't match how YAML naturally expresses time-series data (`- timestamp: ..., value: ...`). +- Don't match how YAML naturally expresses time-series data + (`- timestamp: ..., value: ...`). -This TODO captures the **open question** of how to give mdvs a first-class way to express Array-of-structured-item that: +This TODO captures the **open question** of how to give mdvs a first-class way +to express Array-of-structured-item that: -1. Doesn't reintroduce inline `Object{...}` definitions on disk (Wave C philosophy). +1. Doesn't reintroduce inline `Object{...}` definitions on disk (Wave C + philosophy). 2. Composes with dotted-name flattening. 3. Is opt-in (parallel arrays remain the default fallback). -**Status:** todo, low priority. No users are blocked on it. Captured here so the design space stays open and the constraints are documented. +**Status:** todo, low priority. No users are blocked on it. Captured here so the +design space stays open and the constraints are documented. ## Design candidates (none chosen) @@ -53,7 +63,9 @@ name = "readings" type = "Array(@SensorReading)" ``` -**Rejected** because the definition itself contains an inline `Object{...}` — exactly the form Wave C eliminated for top-level fields. Saying "inline structs are bad at the top but fine in `[definitions]`" is inconsistent. +**Rejected** because the definition itself contains an inline `Object{...}` — +exactly the form Wave C eliminated for top-level fields. Saying "inline structs +are bad at the top but fine in `[definitions]`" is inconsistent. ### Candidate B — Per-element dotted naming @@ -69,15 +81,21 @@ name = "measurements[].value" type = "Float" ``` -The `[]` marker indicates an element-wise grouping. YAML's per-item structure is recovered by the translator: `measurements` is `Array(Object{timestamp, value})` in the JSON Schema, but neither the inline form nor a `$defs` entry is needed in `mdvs.toml`. +The `[]` marker indicates an element-wise grouping. YAML's per-item structure is +recovered by the translator: `measurements` is `Array(Object{timestamp, value})` +in the JSON Schema, but neither the inline form nor a `$defs` entry is needed in +`mdvs.toml`. -**Pros:** consistent with dotted-leaf flattening. No new section. Composes naturally — `measurements[].calibration.baseline.wavelength` works. +**Pros:** consistent with dotted-leaf flattening. No new section. Composes +naturally — `measurements[].calibration.baseline.wavelength` works. -**Cons:** the `[]` marker is syntactically unusual; needs careful spec'ing. Arrays-of-arrays would require `[][]` or similar. +**Cons:** the `[]` marker is syntactically unusual; needs careful spec'ing. +Arrays-of-arrays would require `[][]` or similar. ### Candidate C — `[arrays.]` sections -A new table where each entry describes one structured array, listing its scalar leaves: +A new table where each entry describes one structured array, listing its scalar +leaves: ```toml [arrays.measurements] @@ -85,39 +103,55 @@ timestamp = "String" value = "Float" ``` -**Pros:** no inline structs anywhere; explicit, scannable. Each entry stays scalar-only. +**Pros:** no inline structs anywhere; explicit, scannable. Each entry stays +scalar-only. -**Cons:** introduces a second concept ("fields vs arrays") alongside `[[fields.field]]`; constraints/allowed/required/preprocess machinery would have to be duplicated or generalized. +**Cons:** introduces a second concept ("fields vs arrays") alongside +`[[fields.field]]`; constraints/allowed/required/preprocess machinery would have +to be duplicated or generalized. ### Candidate D — Status quo (parallel arrays forever) -Don't add a representation. Users decompose into parallel arrays when they need it. Accept the loss of per-item grouping at the schema level. +Don't add a representation. Users decompose into parallel arrays when they need +it. Accept the loss of per-item grouping at the schema level. **Pros:** zero work. Consistent with the simplicity goal. -**Cons:** the "no per-item grouping" problem is real for users with structured time-series data. +**Cons:** the "no per-item grouping" problem is real for users with structured +time-series data. ## Constraints any solution must satisfy 1. **No inline `Object{...}` on disk** — Wave C consistency. -2. **Element types must be scalars** — Array-of-Array stays disallowed (already true today). -3. **Must compose with constraints** — each element-leaf can carry its own `pattern`, `min`/`max`, etc. -4. **Must compose with path-scoping** — `allowed`/`required` apply to the whole structured array as a unit, not per-leaf. -5. **JSON Schema round-trip must work** — `export-jsonschema` produces standard `items` schemas; `init --from-jsonschema` reads them back. +2. **Element types must be scalars** — Array-of-Array stays disallowed (already + true today). +3. **Must compose with constraints** — each element-leaf can carry its own + `pattern`, `min`/`max`, etc. +4. **Must compose with path-scoping** — `allowed`/`required` apply to the whole + structured array as a unit, not per-leaf. +5. **JSON Schema round-trip must work** — `export-jsonschema` produces standard + `items` schemas; `init --from-jsonschema` reads them back. ## Decision deferred to when users ask -This TODO stays open with no recommended candidate until concrete usage pressure surfaces. The default workaround (parallel arrays) is enough for example_kb and likely enough for most users. When someone reports a real use case where parallel arrays hurt, we revisit and pick a candidate. +This TODO stays open with no recommended candidate until concrete usage pressure +surfaces. The default workaround (parallel arrays) is enough for example_kb and +likely enough for most users. When someone reports a real use case where +parallel arrays hurt, we revisit and pick a candidate. ## Out of scope (forever, not just v0) - Recursive type definitions (cycle in shape graph). - Remote `$ref` (other files, URLs). -- Definitions with their own constraints (e.g. a definition holding `categories`). +- Definitions with their own constraints (e.g. a definition holding + `categories`). - Auto-extraction of structurally-duplicated shapes during `mdvs init`. ## Related -- [TODO-0097](TODO-0097.md) — Wave C flattening that made this question concrete. -- [TODO-0149](TODO-0149.md) — JSON Schema engine that made `$ref` cheap (and thus tempted candidate A). -- [TODO-0155](TODO-0155.md) — the immediate predecessor; rejects `Array(Object{...})` and points users here for the workaround. +- [TODO-0097](TODO-0097.md) — Wave C flattening that made this question + concrete. +- [TODO-0149](TODO-0149.md) — JSON Schema engine that made `$ref` cheap (and + thus tempted candidate A). +- [TODO-0155](TODO-0155.md) — the immediate predecessor; rejects + `Array(Object{...})` and points users here for the workaround. diff --git a/docs/spec/todos/TODO-0157.md b/docs/spec/todos/TODO-0157.md index fd378d3..4468bc6 100644 --- a/docs/spec/todos/TODO-0157.md +++ b/docs/spec/todos/TODO-0157.md @@ -12,35 +12,35 @@ blocks: [] ## Summary -After [TODO-0016](TODO-0016.md) lands the Lance + LanceDB backend, `build` -will rebuild the IVF-PQ vector index from scratch on every run (Option A: -simple, cheap at our scale). This TODO tracks the eventual move to Option B: -incrementally fold newly added/changed rows into the existing index via -Lance's `optimize` (`OptimizeAction::Index`), retraining the IVF centroids -fully only when a drift threshold is crossed. +After [TODO-0016](TODO-0016.md) lands the Lance + LanceDB backend, `build` will +rebuild the IVF-PQ vector index from scratch on every run (Option A: simple, +cheap at our scale). This TODO tracks the eventual move to Option B: +incrementally fold newly added/changed rows into the existing index via Lance's +`optimize` (`OptimizeAction::Index`), retraining the IVF centroids fully only +when a drift threshold is crossed. ## Details ### Background -LanceDB decouples the data (versioned fragments) from the ANN index built on -top of it. When rows are appended, they are not automatically in the index; -queries stay correct because Lance flat-scans the unindexed fragments and -merges them with ANN results — but those rows are brute-forced until indexed. +LanceDB decouples the data (versioned fragments) from the ANN index built on top +of it. When rows are appended, they are not automatically in the index; queries +stay correct because Lance flat-scans the unindexed fragments and merges them +with ANN results — but those rows are brute-forced until indexed. -TODO-0016 ships **Option A**: after each incremental `build`, rebuild the -vector index outright. At mdvs's scale (hundreds to low-thousands of files) -this costs seconds and always yields optimal recall. Embeddings are not -recomputed — only the index structure is rebuilt, on top of the -already-incremental table writes (`merge_insert` / `delete` for changed files). +TODO-0016 ships **Option A**: after each incremental `build`, rebuild the vector +index outright. At mdvs's scale (hundreds to low-thousands of files) this costs +seconds and always yields optimal recall. Embeddings are not recomputed — only +the index structure is rebuilt, on top of the already-incremental table writes +(`merge_insert` / `delete` for changed files). ### Option B (this TODO) Replace the unconditional index rebuild with: 1. **Incremental optimize** — after writing changed rows, call Lance's - `optimize` with `OptimizeAction::Index` to add only the new fragments to - the existing index. No IVF centroid retraining. + `optimize` with `OptimizeAction::Index` to add only the new fragments to the + existing index. No IVF centroid retraining. 2. **Threshold-triggered full retrain** — when the fraction of unindexed (or tombstoned) rows exceeds a threshold (e.g. 20%), fall back to a full `create_index` to keep recall and centroid quality high. @@ -58,10 +58,10 @@ Replace the unconditional index rebuild with: ### Why deferred -Option A is correct and fast enough for the launch milestone. Option B only -pays off once index rebuild time becomes a noticeable fraction of `build` time -— i.e. at a corpus size mdvs does not yet target. Revisit if Refractions-scale -(or larger) builds feel slow. +Option A is correct and fast enough for the launch milestone. Option B only pays +off once index rebuild time becomes a noticeable fraction of `build` time — i.e. +at a corpus size mdvs does not yet target. Revisit if Refractions-scale (or +larger) builds feel slow. ### Files likely touched diff --git a/docs/spec/todos/TODO-0158.md b/docs/spec/todos/TODO-0158.md index c2ef5c2..69818df 100644 --- a/docs/spec/todos/TODO-0158.md +++ b/docs/spec/todos/TODO-0158.md @@ -12,11 +12,11 @@ blocks: [] ## Summary -The schema-aware `--where` translator (`cmd/search.rs` → `index/backend.rs:: -translate_where_to_struct`, TODO-0016 wave 2) rewrites bare identifier chains -to `data.` using a regex over `[A-Za-z_][A-Za-z0-9_]*`. It does not -understand SQL **quoted identifiers** (`"lab section"`) or field names -containing spaces, quotes, or other punctuation. Such a clause silently +The schema-aware `--where` translator (`cmd/search.rs` → +`index/backend.rs:: translate_where_to_struct`, TODO-0016 wave 2) rewrites bare +identifier chains to `data.` using a regex over `[A-Za-z_][A-Za-z0-9_]*`. +It does not understand SQL **quoted identifiers** (`"lab section"`) or field +names containing spaces, quotes, or other punctuation. Such a clause silently matches nothing instead of filtering correctly. ## Details @@ -28,8 +28,8 @@ example_kb has fields with exotic names: `lab section` (space), `notes"v2"` mdvs search q --where '"lab section" = "A"' ``` -produces a translation like `"data.lab data.section" = ...` (the inner words -are matched and prefixed independently; the surrounding double quotes are not +produces a translation like `"data.lab data.section" = ...` (the inner words are +matched and prefixed independently; the surrounding double quotes are not recognized as a quoted-identifier delimiter), which references no real column and returns zero hits — with no error. @@ -37,19 +37,18 @@ and returns zero hits — with no error. - The identifier regex matches `[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_]...)*` and has no notion of double-quoted identifiers. -- Single-quoted **string literals** are protected (skipped), but - double-quoted **identifiers** are not. +- Single-quoted **string literals** are protected (skipped), but double-quoted + **identifiers** are not. ### Possible fixes -- Extend the literal/skip pass to also recognize double-quoted identifiers, - and when one wraps a frontmatter field, rewrite to ``data."lab section"`` (or - the LanceDB-accepted nested form) rather than tokenizing the inside. -- Verify what LanceDB/DataFusion accepts for struct-child access when the - child name contains spaces (likely backtick or double-quote quoting of the - segment — spike it). -- Add translator unit tests + an integration test on a field named with a - space. +- Extend the literal/skip pass to also recognize double-quoted identifiers, and + when one wraps a frontmatter field, rewrite to `data."lab section"` (or the + LanceDB-accepted nested form) rather than tokenizing the inside. +- Verify what LanceDB/DataFusion accepts for struct-child access when the child + name contains spaces (likely backtick or double-quote quoting of the segment — + spike it). +- Add translator unit tests + an integration test on a field named with a space. ### Why low priority diff --git a/docs/spec/todos/TODO-0159.md b/docs/spec/todos/TODO-0159.md index e6ff12a..ec4032a 100644 --- a/docs/spec/todos/TODO-0159.md +++ b/docs/spec/todos/TODO-0159.md @@ -21,19 +21,40 @@ files_removed: Two parts, both shipped 2026-05-25. -**1. Mitigation (mdvs-side, commit `7446289`).** `translate_where_to_struct` in `crates/mdvs/src/index/backend.rs` now refuses `--where` clauses that reference an `Array(Float)` field, returning a clear error before LanceDB sees the clause: +**1. Mitigation (mdvs-side, commit `7446289`).** `translate_where_to_struct` in +`crates/mdvs/src/index/backend.rs` now refuses `--where` clauses that reference +an `Array(Float)` field, returning a clear error before LanceDB sees the clause: ``` filtering on Array(Float) field '' is not supported in --where. Filter on a different field or store the values in a parallel scalar field. ``` -Covered by translator unit tests + `integration_array_float_filter_errors_not_panics` across all three search modes. - -**2. Upstream filing — [lancedb/lancedb#3446](https://github.com/lancedb/lancedb/issues/3446).** *`bug(rust): v2.1 — scanning List panics when batch was loaded from Arrow IPC`*. Filed with a fully self-contained 90-line `rust-script` spike + a 2 KB Arrow IPC blob bundled as a zip attachment, plus a 5-point bisection summary documenting which axes were tested. References lancedb#3194 (`bug(python): v2.2: list read panic with nullable data across multiple fragments`) as the closest-known related issue — same family, different lance-encoding version / storage version / float type / panic site. - -**Followup when upstream fixes the panic:** remove the `float_list_fields` parameter from `translate_where_to_struct` and the early-rejection branch (also the `integration_array_float_filter_errors_not_panics` test that asserts the error). The mitigation is forward-compatible — it produces a clean error rather than working — so leaving it in place after the fix lands is harmless if forgotten, just over-conservative. - -**Reproducer artifacts removed from the repo (2026-05-25).** `scripts/test_lance_standalone_repro.rs` and `scripts/test_lance_mv_only.arrow` were the reproducer's home before filing. The frozen version now lives inside the body and attachment of lancedb#3446. If the bug is ever re-tested locally (after a future lancedb bump), regenerate the artifacts from the issue body. +Covered by translator unit tests + +`integration_array_float_filter_errors_not_panics` across all three search +modes. + +**2. Upstream filing — +[lancedb/lancedb#3446](https://github.com/lancedb/lancedb/issues/3446).** +_`bug(rust): v2.1 — scanning List panics when batch was loaded from Arrow IPC`_. +Filed with a fully self-contained 90-line `rust-script` spike + a 2 KB Arrow IPC +blob bundled as a zip attachment, plus a 5-point bisection summary documenting +which axes were tested. References lancedb#3194 +(`bug(python): v2.2: list read panic with nullable data across multiple fragments`) +as the closest-known related issue — same family, different lance-encoding +version / storage version / float type / panic site. + +**Followup when upstream fixes the panic:** remove the `float_list_fields` +parameter from `translate_where_to_struct` and the early-rejection branch (also +the `integration_array_float_filter_errors_not_panics` test that asserts the +error). The mitigation is forward-compatible — it produces a clean error rather +than working — so leaving it in place after the fix lands is harmless if +forgotten, just over-conservative. + +**Reproducer artifacts removed from the repo (2026-05-25).** +`scripts/test_lance_standalone_repro.rs` and `scripts/test_lance_mv_only.arrow` +were the reproducer's home before filing. The frozen version now lives inside +the body and attachment of lancedb#3446. If the bug is ever re-tested locally +(after a future lancedb bump), regenerate the artifacts from the issue body. --- @@ -42,41 +63,40 @@ The original investigation log is preserved below for historical context. ## Status **Mitigated 2026-05-23.** The translator now rejects `--where` references to -`Array(Float)` (`List`) fields with a clean, actionable error across -all modes (`7446289`). The underlying Lance bug remains; revisit on a -`lancedb` bump. Priority lowered from `medium` to `low` since the -user-visible severity (crash/hang → clear error) is fixed. +`Array(Float)` (`List`) fields with a clean, actionable error across all +modes (`7446289`). The underlying Lance bug remains; revisit on a `lancedb` +bump. Priority lowered from `medium` to `low` since the user-visible severity +(crash/hang → clear error) is fixed. ### Bump test (2026-05-23) — no quick fix available upstream - `lancedb 0.29` is the newest crates.io release; no patch-version bump. - The lance family is at `6.0.1` (released 2026-05-20). Tried forcing it via `[patch.crates-io] { git = ".../lancedb/lance", tag = "v6.0.1" }`: cargo - refused — lancedb 0.29 hard-pins `lance = "=6.0.0"`, so 6.0.1 doesn't - satisfy the requirement and the patches show as "not used in the crate - graph". Would need to fork lancedb and relax the pin. -- Even if forced, `lance-encoding 6.0.1`'s `slice_next_task` *still* has the + refused — lancedb 0.29 hard-pins `lance = "=6.0.0"`, so 6.0.1 doesn't satisfy + the requirement and the patches show as "not used in the crate graph". Would + need to fork lancedb and relax the pin. +- Even if forced, `lance-encoding 6.0.1`'s `slice_next_task` _still_ has the same `self.data.front_mut().unwrap()` textually (180-line diff in - `primitive.rs` vs 6.0.0, but that line is unchanged). The fix is not in - the 6.x line yet. -- `lancedb` `main` is `0.30.0-beta.1`, depending on `lance 7.0.0-beta.13` - via the *new* `github.com/lance-format/lance` repo. Major-version jump; - would likely require mdvs API updates. + `primitive.rs` vs 6.0.0, but that line is unchanged). The fix is not in the + 6.x line yet. +- `lancedb` `main` is `0.30.0-beta.1`, depending on `lance 7.0.0-beta.13` via + the _new_ `github.com/lance-format/lance` repo. Major-version jump; would + likely require mdvs API updates. Path forward: wait for `lancedb 0.30` stable on lance 7.x, then plan a coordinated bump (separate work). The bisection spike (`scripts/test_lance_float_list_filter.rs`) and side-by-side -(`scripts/test_lance_vs_datafusion_float_list.rs`) remain the re-test -harness. +(`scripts/test_lance_vs_datafusion_float_list.rs`) remain the re-test harness. ## Summary A `--where` clause that references an `Array(Float)` (Arrow `List`) -frontmatter field panics inside LanceDB's encoding layer. In semantic mode -the panic surfaces on `main` (process exits). In fulltext/hybrid mode it -panics on a `tokio-rt-worker`, leaving the main task awaiting a future that -never resolves — **the process hangs**. Array(String) fields are unaffected; -normal searches (no such filter) are fine because the column is not projected. +frontmatter field panics inside LanceDB's encoding layer. In semantic mode the +panic surfaces on `main` (process exits). In fulltext/hybrid mode it panics on a +`tokio-rt-worker`, leaving the main task awaiting a future that never resolves — +**the process hangs**. Array(String) fields are unaffected; normal searches (no +such filter) are fine because the column is not projected. ## Details @@ -100,12 +120,11 @@ called `Option::unwrap()` on a `None` value - Triggers for **any** `--where` predicate that references a `List` field (membership, null checks — anything that makes Lance scan/decode the column to evaluate the filter). -- `Array(String)` (`List`, e.g. `tags`, `measurement_timestamps`) - filters work fine (`array_has(tags, 'x')` → correct hits). -- `build`, `check`, and normal `search` (no float-array filter) are - unaffected — search projects only - `[file_id, filepath, start_line, end_line, chunk_text]`, so the float-array - column is only decoded when a `--where` references it. +- `Array(String)` (`List`, e.g. `tags`, `measurement_timestamps`) filters + work fine (`array_has(tags, 'x')` → correct hits). +- `build`, `check`, and normal `search` (no float-array filter) are unaffected — + search projects only `[file_id, filepath, start_line, end_line, chunk_text]`, + so the float-array column is only decoded when a `--where` references it. - The `embedding` column is `FixedSizeList` and is fine — this is specific to the variable-length `List` decode path. @@ -118,12 +137,12 @@ A LanceDB / lance-encoding 6.0 bug decoding `List` (nested in the 1. **Upstream / version bump** — retest after a `lancedb` upgrade; this may be fixed in a newer lance-encoding. Lowest effort, just gated on a bump. -2. **Schema-aware guard (mitigation)** — the `--where` translator already - knows the frontmatter field names (data Struct children); extend it to also - know their types and return a clean error - ("filtering on Array(Float) fields is not supported — LanceDB limitation") - when a clause references an `Array(Float)` field, turning the panic into a - graceful user error. Avoids the crash without fixing Lance. +2. **Schema-aware guard (mitigation)** — the `--where` translator already knows + the frontmatter field names (data Struct children); extend it to also know + their types and return a clean error ("filtering on Array(Float) fields is + not supported — LanceDB limitation") when a clause references an + `Array(Float)` field, turning the panic into a graceful user error. Avoids + the crash without fixing Lance. 3. **Storage change** — store float arrays differently (e.g. fixed-size or a different encoding). Heavy; not worth it for a niche filter. @@ -134,8 +153,8 @@ Recommended: try (1) on the next lancedb bump; (2) is already in place. `index/backend.rs` learned `float_list_child_names(schema)`, which scans the `data` Struct for children of type `List`. `translate_where_to_struct` takes that set as a new param and returns an `Err` early if any matched -identifier in the clause names one of those fields (handling both bare names -and pre-qualified `data.` references). The user sees: +identifier in the clause names one of those fields (handling both bare names and +pre-qualified `data.` references). The user sees: ``` Error: filtering on Array(Float) field 'measurement_values' is not supported @@ -150,8 +169,8 @@ instead of a crash/hang. Covered by translator unit tests + - (upstream) `lance-encoding` `FixedFullZipDecoder::slice_next_task` at `primitive.rs:2505` (drain of a `StructuralListDecoder` over a sparse - `List`). Stack: `slice_next_task → drain → drain` then up through - the structural list decoder. + `List`). Stack: `slice_next_task → drain → drain` then up through the + structural list decoder. - (mdvs cleanup, once Lance is fixed) drop `float_list_child_names` + the early-error in `translate_where_to_struct`, and remove the integration test asserting the error. @@ -167,68 +186,70 @@ The trigger is precisely two conditions, both required: 1. **At most 2 non-null rows** in `data.measurement_values`. 2. **At least one non-null row's list has length ≤ 3.** -Matrix observed (all on v2.1 file format, 79-row dataset, mdvs build with -the unchanged `mdvs.toml`): - -| `measurement_values` lengths | Non-null rows | Result | Panic | -|---|---|---|---| -| [3, 2] (baseline) | 2 | PANIC | `lance-encoding/.../primitive.rs:2505` unwrap | -| [3, 3] | 2 | PANIC | same | -| [1, 1] | 2 | PANIC | `lance-encoding/.../buffer.rs:290` slice OOB | -| [3] (one file only) | 1 | PANIC | same | -| [2, 5] | 2 | PANIC | same | -| [3, 4] | 2 | PANIC | same | -| [4, 4] | 2 | **OK** | — | -| [4, 5] | 2 | **OK** | — | -| [30, 30] | 2 | **OK** | — | -| [3, 3, 3] (three files) | 3 | **OK** | — | +Matrix observed (all on v2.1 file format, 79-row dataset, mdvs build with the +unchanged `mdvs.toml`): + +| `measurement_values` lengths | Non-null rows | Result | Panic | +| ---------------------------- | ------------- | ------ | --------------------------------------------- | +| [3, 2] (baseline) | 2 | PANIC | `lance-encoding/.../primitive.rs:2505` unwrap | +| [3, 3] | 2 | PANIC | same | +| [1, 1] | 2 | PANIC | `lance-encoding/.../buffer.rs:290` slice OOB | +| [3] (one file only) | 1 | PANIC | same | +| [2, 5] | 2 | PANIC | same | +| [3, 4] | 2 | PANIC | same | +| [4, 4] | 2 | **OK** | — | +| [4, 5] | 2 | **OK** | — | +| [30, 30] | 2 | **OK** | — | +| [3, 3, 3] (three files) | 3 | **OK** | — | Interpretation: lance-encoding 6.0 picks a structural-list encoder strategy based on per-page statistics; the `≤ 2 non-null rows with ≤ 3 values` branch -hits a decoder path with at least two distinct unguarded `None` unwraps / -buffer slices. Three or more non-null rows, or all rows ≥ 4 values, route -to a different encoder/decoder that has no bug. +hits a decoder path with at least two distinct unguarded `None` unwraps / buffer +slices. Three or more non-null rows, or all rows ≥ 4 values, route to a +different encoder/decoder that has no bug. The minimum reproducer remains the on-disk example_kb dataset -(`scripts/test_lance_repro_from_disk.rs`); a fully programmatic Lance-API -spike with the same in-memory shape (59 rows, Struct of 36 children, -2 non-null `List` of lengths 3 and 2 at positions 36–37, -companion FixedSizeList, FTS index) does **not** panic, -suggesting the trigger also depends on lancedb's writer-side defaults -(batch boundaries, `create_table` vs direct `Dataset::write`) that -direct-API spikes don't replicate. Open question for the upstream issue: -which writer-side knob nudges the encoder onto the buggy branch. +(`scripts/test_lance_repro_from_disk.rs`); a fully programmatic Lance-API spike +with the same in-memory shape (59 rows, Struct of 36 children, 2 non-null +`List` of lengths 3 and 2 at positions 36–37, companion +FixedSizeList, FTS index) does **not** panic, suggesting the +trigger also depends on lancedb's writer-side defaults (batch boundaries, +`create_table` vs direct `Dataset::write`) that direct-API spikes don't +replicate. Open question for the upstream issue: which writer-side knob nudges +the encoder onto the buggy branch. ## Final isolation (2026-05-24) Earlier source-data bisection was misleading: the rule "≤ 2 non-null rows AND list length ≤ 3" only holds when the data is built and written through mdvs. Further synthetic spikes (writing the **same** logical data via vanilla -`lancedb::create_table` from freshly built Arrow arrays) did NOT trigger the -bug — across single-column tables, full-schema matches, and even round-trips -of example_kb's actual rows reconstructed via lancedb's decoder. +`lancedb::create_table` from freshly built Arrow arrays) did NOT trigger the bug +— across single-column tables, full-schema matches, and even round-trips of +example_kb's actual rows reconstructed via lancedb's decoder. -Decisive isolation came from a different angle: dump mdvs's exact Arrow batch -to an Arrow IPC file before calling `lancedb::create_table`, then replay that -IPC bytes via a vanilla 50-line lancedb script. Replaying does panic. Then -bisecting the replayed batch: +Decisive isolation came from a different angle: dump mdvs's exact Arrow batch to +an Arrow IPC file before calling `lancedb::create_table`, then replay that IPC +bytes via a vanilla 50-line lancedb script. Replaying does panic. Then bisecting +the replayed batch: - Every top-level column except `data` can be dropped → still panics. -- Every sibling child of `data.measurement_values` can be dropped → still panics. +- Every sibling child of `data.measurement_values` can be dropped → still + panics. - Schema metadata stripped → still panics. -- Reduced to two columns (`id Int32` + `data Struct{measurement_values List}`), - the resulting batch fits in a **2 KB Arrow IPC blob** that still panics on replay. +- Reduced to two columns (`id Int32` + + `data Struct{measurement_values List}`), the resulting batch fits in + a **2 KB Arrow IPC blob** that still panics on replay. Then the key step: rebuilding the `measurement_values` ListArray via -`ListBuilder` with the same logical values → **OK**. -Cloning the IPC-loaded Arrow Buffer's bytes into fresh `Vec` allocations -and rebuilding the ArrayData → **OK**. +`ListBuilder` with the same logical values → **OK**. Cloning the +IPC-loaded Arrow Buffer's bytes into fresh `Vec` allocations and rebuilding +the ArrayData → **OK**. So the trigger is **a memory-layout property of the Arrow Buffer**, not the -logical content of the array or its schema. Bytes are byte-identical between -the IPC-loaded and freshly-built arrays; the difference is what backs the -`arrow_buffer::Buffer` (an Arc-shared `Bytes` slice carved out of one large -IPC body allocation vs a fresh small heap allocation). Lance's writer picks a +logical content of the array or its schema. Bytes are byte-identical between the +IPC-loaded and freshly-built arrays; the difference is what backs the +`arrow_buffer::Buffer` (an Arc-shared `Bytes` slice carved out of one large IPC +body allocation vs a fresh small heap allocation). Lance's writer picks a different encoder branch based on that property, and the resulting on-disk encoding panics on read. @@ -237,18 +258,17 @@ Self-contained reproducer artifacts (filed inside the body and attachment of under `scripts/` were removed when the issue was filed, since the GitHub issue is now the authoritative archive): -- A ~90-line pure-lancedb `rust-script` spike — no mdvs runtime, no - example_kb directory. Embedded the IPC blob via `include_bytes!`. +- A ~90-line pure-lancedb `rust-script` spike — no mdvs runtime, no example_kb + directory. Embedded the IPC blob via `include_bytes!`. - A 2 KB Arrow IPC file (one batch, two columns, 79 rows). -Panic site: -`lance-encoding-6.0.0/src/encodings/logical/primitive.rs:2505` -(`FixedFullZipDecoder::slice_next_task`, `self.data.front_mut().unwrap()` -on an empty VecDeque — writer/decoder state inconsistency). +Panic site: `lance-encoding-6.0.0/src/encodings/logical/primitive.rs:2505` +(`FixedFullZipDecoder::slice_next_task`, `self.data.front_mut().unwrap()` on an +empty VecDeque — writer/decoder state inconsistency). -What we did NOT determine, and the upstream issue should ask about: -which specific `arrow_buffer::Buffer` property is being read by lance's writer -that differs between IPC-backed slices and fresh-heap buffers. Candidates: -`Bytes` deallocator type, ptr offset within the underlying allocation, +What we did NOT determine, and the upstream issue should ask about: which +specific `arrow_buffer::Buffer` property is being read by lance's writer that +differs between IPC-backed slices and fresh-heap buffers. Candidates: `Bytes` +deallocator type, ptr offset within the underlying allocation, capacity-vs-length ratio, alignment. A pure-`Vec` arena reproduction with matched ptr offsets + capacity + alignment did not trigger. diff --git a/docs/spec/todos/TODO-0160.md b/docs/spec/todos/TODO-0160.md index 07da637..66a7214 100644 --- a/docs/spec/todos/TODO-0160.md +++ b/docs/spec/todos/TODO-0160.md @@ -12,53 +12,85 @@ blocks: [] ## Summary -Add an optional time-decay factor to search ranking so that more-recent notes float to the top relative to older ones with similar relevance. Borrowed from `markdown-vdb`'s recency-weighted ranking. Skipped for v0.6.0 launch — open for later. +Add an optional time-decay factor to search ranking so that more-recent notes +float to the top relative to older ones with similar relevance. Borrowed from +`markdown-vdb`'s recency-weighted ranking. Skipped for v0.6.0 launch — open for +later. ## Motivation -For note-taking workflows ("what was I working on last week?") and meeting/log vaults, recency is signal. A note from two days ago that's a 0.6 cosine match is often more useful than a perfect-cosine note from three years ago. Currently mdvs ranks purely by mode-specific score; ages don't enter the formula. +For note-taking workflows ("what was I working on last week?") and meeting/log +vaults, recency is signal. A note from two days ago that's a 0.6 cosine match is +often more useful than a perfect-cosine note from three years ago. Currently +mdvs ranks purely by mode-specific score; ages don't enter the formula. -`markdown-vdb` exposes this as `--decay`, `--decay-half-life`, `--decay-include PATTERN`, `--decay-exclude PATTERN` on its search command. The shape is reasonable; we should design our own surface that fits mdvs's conventions. +`markdown-vdb` exposes this as `--decay`, `--decay-half-life`, +`--decay-include PATTERN`, `--decay-exclude PATTERN` on its search command. The +shape is reasonable; we should design our own surface that fits mdvs's +conventions. ## Open design questions (decide before implementation) 1. **Where does "now" come from?** Candidates: - - The file's `built_at` (already on every chunk row — easiest, but lies about content age when builds are infrequent). - - The file's `content_hash` last-changed time (we'd need to start tracking this; today we just compare hashes for incremental builds). - - A `date` / `created` / `synced_at` field in the user's frontmatter (requires schema awareness — only works if the field exists and is typed `Date` or `DateTime`). - - `mtime` of the source file on disk (cheap; survives across builds; doesn't survive `git clone`). - - Realistically: prefer a configured frontmatter field (typed Date/DateTime), fall back to `built_at`. Make the field configurable in `[search]`. + - The file's `built_at` (already on every chunk row — easiest, but lies about + content age when builds are infrequent). + - The file's `content_hash` last-changed time (we'd need to start tracking + this; today we just compare hashes for incremental builds). + - A `date` / `created` / `synced_at` field in the user's frontmatter + (requires schema awareness — only works if the field exists and is typed + `Date` or `DateTime`). + - `mtime` of the source file on disk (cheap; survives across builds; doesn't + survive `git clone`). + + Realistically: prefer a configured frontmatter field (typed Date/DateTime), + fall back to `built_at`. Make the field configurable in `[search]`. 2. **Per-query flag vs config-level toggle vs both?** - - Per-query `--decay-half-life 30d` → user opts in each call. Simple, explicit. - - `[search].decay_half_life = "30d"` → always on once configured, can be overridden with `--no-decay`. More ergonomic for daily use. + - Per-query `--decay-half-life 30d` → user opts in each call. Simple, + explicit. + - `[search].decay_half_life = "30d"` → always on once configured, can be + overridden with `--no-decay`. More ergonomic for daily use. - Both is the obvious answer; config sets the default, flag overrides. 3. **Which mode(s) does it apply to?** - - All modes? Hybrid only? Whatever scores LanceDB produces are not on the same scale across modes, so applying the same decay multiplier to a cosine score (0–1) vs a BM25 score (unbounded) vs an RRF score (tiny) needs care. - - Simplest: normalise the score to a 0–1 percentile within the candidate set, then multiply by `exp(-Δt / λ)` where λ = half-life × ln(2). Mode-agnostic. + - All modes? Hybrid only? Whatever scores LanceDB produces are not on the + same scale across modes, so applying the same decay multiplier to a cosine + score (0–1) vs a BM25 score (unbounded) vs an RRF score (tiny) needs care. + - Simplest: normalise the score to a 0–1 percentile within the candidate set, + then multiply by `exp(-Δt / λ)` where λ = half-life × ln(2). Mode-agnostic. 4. **Scope: include / exclude path patterns?** - - markdown-vdb has `--decay-include` and `--decay-exclude` so reference material doesn't get penalised. Likely a `[search].decay_exclude = ["reference/**", "archive/**"]` config knob, no per-query flag. + - markdown-vdb has `--decay-include` and `--decay-exclude` so reference + material doesn't get penalised. Likely a + `[search].decay_exclude = ["reference/**", "archive/**"]` config knob, no + per-query flag. 5. **What's the default half-life if the user just says `--decay`?** - - markdown-vdb defaults to 30 days. Same default is probably fine for note-taking; longer for technical-doc vaults. Configurable. + - markdown-vdb defaults to 30 days. Same default is probably fine for + note-taking; longer for technical-doc vaults. Configurable. 6. **JSON output shape.** - - Either add a `decay_factor` field per hit alongside `score`, or expose two scores (`score_raw`, `score_decayed`). The latter is more honest because it lets users see what the decay actually did. + - Either add a `decay_factor` field per hit alongside `score`, or expose two + scores (`score_raw`, `score_decayed`). The latter is more honest because it + lets users see what the decay actually did. ## Acceptance criteria (provisional) -- `mdvs search "query" --decay` applies time-decay using the configured field and half-life, or sensible defaults. -- `[search]` config supports `decay_field`, `decay_half_life`, `decay_exclude`, `decay_default` (bool). -- A test confirms: two hits with similar raw scores, one recent and one old → recent ranks higher. -- A test confirms: a hit covered by `decay_exclude` is not penalised even when old. +- `mdvs search "query" --decay` applies time-decay using the configured field + and half-life, or sensible defaults. +- `[search]` config supports `decay_field`, `decay_half_life`, `decay_exclude`, + `decay_default` (bool). +- A test confirms: two hits with similar raw scores, one recent and one old → + recent ranks higher. +- A test confirms: a hit covered by `decay_exclude` is not penalised even when + old. - JSON output exposes both raw and decayed scores. - Documented in `book/src/concepts/search.md` or a new `search-recency.md` page. ## Out of scope -- Trying to infer the "right" date field automatically. The user picks it in `[search]`. +- Trying to infer the "right" date field automatically. The user picks it in + `[search]`. - `mtime`-based decay. Too fragile (clones, copies, sync tools reset it). -- Re-ranking the BM25 / vector index itself — decay is applied at result-assembly time, after LanceDB returns candidates. +- Re-ranking the BM25 / vector index itself — decay is applied at + result-assembly time, after LanceDB returns candidates. diff --git a/docs/spec/todos/TODO-0161.md b/docs/spec/todos/TODO-0161.md index c152fa6..2656f54 100644 --- a/docs/spec/todos/TODO-0161.md +++ b/docs/spec/todos/TODO-0161.md @@ -12,71 +12,122 @@ blocks: [] ## Summary -The current asciinema demo (`assets/demo.py` → `assets/demo.cast` → `assets/demo.gif`) is functional but cluttered: there are no breaks between sections, the deliberately-broken note triggers too many violations at once, the commands have no syntax highlighting, and long output scrolls the command out of view. Pre-launch polish before we publicise the demo any wider. +The current asciinema demo (`assets/demo.py` → `assets/demo.cast` → +`assets/demo.gif`) is functional but cluttered: there are no breaks between +sections, the deliberately-broken note triggers too many violations at once, the +commands have no syntax highlighting, and long output scrolls the command out of +view. Pre-launch polish before we publicise the demo any wider. ## Details ### 1. Clear screen between sections (Ctrl-L) -The demo runs `init` → modify a note → `check` → `build` → `search` as one continuous stream. After each major step the previous output stays on screen, so by the end the viewer can't tell where one command ended and the next began. +The demo runs `init` → modify a note → `check` → `build` → `search` as one +continuous stream. After each major step the previous output stays on screen, so +by the end the viewer can't tell where one command ended and the next began. -- Add `comment("…", clear=True)` (or similar) that sends `\x0c` to bash before printing the section header. Or use `printf '\\033[2J\\033[H'` to scroll-and-clear. -- Apply between: init → write-bad-note, check → fix-note, fix-note → build, build → search. The header comment for each section gives the viewer a moment to read it before the next command runs. +- Add `comment("…", clear=True)` (or similar) that sends `\x0c` to bash before + printing the section header. Or use `printf '\\033[2J\\033[H'` to + scroll-and-clear. +- Apply between: init → write-bad-note, check → fix-note, fix-note → build, + build → search. The header comment for each section gives the viewer a moment + to read it before the next command runs. ### 2. Smaller, deliberate violations in the new note -The current demo creates a note with multiple violations at once so the `check` output shows several violation kinds. Result: a wall of red boxes that's hard to follow in 8 seconds of GIF. +The current demo creates a note with multiple violations at once so the `check` +output shows several violation kinds. Result: a wall of red boxes that's hard to +follow in 8 seconds of GIF. -- Pick **one** violation that's instantly readable — e.g. a `status: draft` value where the schema requires `published` / `archived`, producing one `InvalidCategory` box. +- Pick **one** violation that's instantly readable — e.g. a `status: draft` + value where the schema requires `published` / `archived`, producing one + `InvalidCategory` box. - Or two if we want to demonstrate the structure, but no more. -- The point of this section is "mdvs catches a structural mistake a flat linter wouldn't" — one clear example is more persuasive than a wall of errors. -- Update `comment()` text to set the expectation: *"Let's add a note with a typo in the status — watch what mdvs flags."* +- The point of this section is "mdvs catches a structural mistake a flat linter + wouldn't" — one clear example is more persuasive than a wall of errors. +- Update `comment()` text to set the expectation: _"Let's add a note with a typo + in the status — watch what mdvs flags."_ ### 3. Syntax highlighting on terminal commands -Bash echoes commands as plain text. We want commands like `mdvs init example_kb` to render with colour: command name in one shade, arguments in another, strings highlighted. +Bash echoes commands as plain text. We want commands like `mdvs init example_kb` +to render with colour: command name in one shade, arguments in another, strings +highlighted. Options, in order of effort: -- **Easiest**: switch the demo shell from bash to zsh with `zsh-syntax-highlighting` enabled. zsh renders typed-but-not-yet-executed commands with colours from a config file. We'd configure a temporary `~/.zshrc` for the demo session. -- **Middle**: use `bat` to pre-render command snippets with bash highlighting, but `bat` output is text — asciinema would record the rendered colours fine. The flow would be: print colourised command via `bat`, then actually execute it. Two prints per command, but one of them is colourised. -- **Cleanest**: a tiny shell wrapper that takes a command string, prints it with ANSI colour codes (highlight `mdvs`, sub-commands, flags, strings), then runs it. ~30 lines of Python helper. - -The "cleanest" option is probably the right call because we don't want demo behaviour to depend on a specific zsh config or plugin version. +- **Easiest**: switch the demo shell from bash to zsh with + `zsh-syntax-highlighting` enabled. zsh renders typed-but-not-yet-executed + commands with colours from a config file. We'd configure a temporary + `~/.zshrc` for the demo session. +- **Middle**: use `bat` to pre-render command snippets with bash highlighting, + but `bat` output is text — asciinema would record the rendered colours fine. + The flow would be: print colourised command via `bat`, then actually execute + it. Two prints per command, but one of them is colourised. +- **Cleanest**: a tiny shell wrapper that takes a command string, prints it with + ANSI colour codes (highlight `mdvs`, sub-commands, flags, strings), then runs + it. ~30 lines of Python helper. + +The "cleanest" option is probably the right call because we don't want demo +behaviour to depend on a specific zsh config or plugin version. ### 4. Keep the last command visible — don't let it scroll off -This is the hardest. When `mdvs search "..." -v` produces a long verbose output, the command itself scrolls off the top of the terminal. The viewer sees the output but can't tell what produced it. +This is the hardest. When `mdvs search "..." -v` produces a long verbose output, +the command itself scrolls off the top of the terminal. The viewer sees the +output but can't tell what produced it. Options (rough effort gradient): -- **(a) Make the terminal taller**. The current GIF is 110×30. Bumping to 110×40 or 110×50 buys headroom for free. Cheap, but doesn't solve the problem at large widths. -- **(b) Trim long output deliberately.** Pipe everything through `head -N` or `awk` to keep the visible portion small. Cleanest visually, but lies about what mdvs actually outputs. -- **(c) Re-print the command after the output**. After every command finishes, echo the command line again in a dim colour (e.g. `# ran: mdvs search "experiment" -v`). The viewer always sees what produced the output above by glancing down. Easy to implement, slightly redundant but very clear. -- **(d) Use tmux with a fixed status bar** showing the last command. Requires recording the tmux session, not just the bash session. Doable but adds a dependency and complicates the regen workflow. -- **(e) Compute output height per command** and only run commands whose output fits in the remaining terminal height; clear and reprint if it would exceed. - -Discussion: **(c) is probably the best fit**. Re-printing the command in dim text after a long output is what tutorial videos commonly do. It costs nothing in complexity and reads naturally. (a) can be combined with (c) for extra headroom. +- **(a) Make the terminal taller**. The current GIF is 110×30. Bumping to 110×40 + or 110×50 buys headroom for free. Cheap, but doesn't solve the problem at + large widths. +- **(b) Trim long output deliberately.** Pipe everything through `head -N` or + `awk` to keep the visible portion small. Cleanest visually, but lies about + what mdvs actually outputs. +- **(c) Re-print the command after the output**. After every command finishes, + echo the command line again in a dim colour (e.g. + `# ran: mdvs search "experiment" -v`). The viewer always sees what produced + the output above by glancing down. Easy to implement, slightly redundant but + very clear. +- **(d) Use tmux with a fixed status bar** showing the last command. Requires + recording the tmux session, not just the bash session. Doable but adds a + dependency and complicates the regen workflow. +- **(e) Compute output height per command** and only run commands whose output + fits in the remaining terminal height; clear and reprint if it would exceed. + +Discussion: **(c) is probably the best fit**. Re-printing the command in dim +text after a long output is what tutorial videos commonly do. It costs nothing +in complexity and reads naturally. (a) can be combined with (c) for extra +headroom. ### Files to touch - `assets/demo.py` — all four points are demo-script changes -- `assets/README.md` — document the new helpers (highlight, clear, post-echo) and the regen workflow +- `assets/README.md` — document the new helpers (highlight, clear, post-echo) + and the regen workflow - `assets/demo.cast` — regenerate -- `assets/demo.gif` — regenerate (via `agg --theme monokai --font-size 16` per memory) +- `assets/demo.gif` — regenerate (via `agg --theme monokai --font-size 16` per + memory) ### Acceptance criteria - [ ] Each major section (init, check, build, search) starts on a clean screen -- [ ] The deliberate-violation section produces at most 2 clearly distinct violation boxes -- [ ] Commands render with at least three colour roles (binary name, sub-command, string literals) -- [ ] After every command whose output exceeds ~10 lines, the command line is re-echoed in a dim colour -- [ ] GIF total length is ≤ 60s (current is ~50s — adding clears shouldn't blow this) -- [ ] `assets/README.md` documents how to regenerate, including the new dependencies (if any) +- [ ] The deliberate-violation section produces at most 2 clearly distinct + violation boxes +- [ ] Commands render with at least three colour roles (binary name, + sub-command, string literals) +- [ ] After every command whose output exceeds ~10 lines, the command line is + re-echoed in a dim colour +- [ ] GIF total length is ≤ 60s (current is ~50s — adding clears shouldn't blow + this) +- [ ] `assets/README.md` documents how to regenerate, including the new + dependencies (if any) ## Out of scope -- Switching away from asciinema entirely (a different recording tool / a hand-edited video) +- Switching away from asciinema entirely (a different recording tool / a + hand-edited video) - Re-doing the example_kb to be smaller / more aesthetic - Translating the demo to other formats (mp4, webm) - Recording multiple variants for different audiences diff --git a/docs/spec/todos/TODO-0162.md b/docs/spec/todos/TODO-0162.md index a87e9df..2c7cc5b 100644 --- a/docs/spec/todos/TODO-0162.md +++ b/docs/spec/todos/TODO-0162.md @@ -41,7 +41,14 @@ files_updated: ## Summary -mdvs currently parses YAML frontmatter only (hardcoded `Matter::::new()` in `discover/scan.rs:76`). Hugo, MDX/Astro, and several other static-site ecosystems accept TOML (`+++` delimited) and JSON (`{...}` delimited) frontmatter too. `gray_matter` already has both engines available — we just never wired them up. Adding auto-detection plus an explicit config override is small, removes a real adoption barrier (Hugo TOML users can't try mdvs on their existing vault today), and lets us state in marketing that we support "what Hugo supports". Pre-launch. +mdvs currently parses YAML frontmatter only (hardcoded `Matter::::new()` +in `discover/scan.rs:76`). Hugo, MDX/Astro, and several other static-site +ecosystems accept TOML (`+++` delimited) and JSON (`{...}` delimited) +frontmatter too. `gray_matter` already has both engines available — we just +never wired them up. Adding auto-detection plus an explicit config override is +small, removes a real adoption barrier (Hugo TOML users can't try mdvs on their +existing vault today), and lets us state in marketing that we support "what Hugo +supports". Pre-launch. ## Current state @@ -51,22 +58,26 @@ mdvs currently parses YAML frontmatter only (hardcoded `Matter::::new()` i let matter = Matter::::new(); ``` -That's the only engine instance. Bare files (no `---` block) are tolerated; non-YAML frontmatter is either silently treated as part of the body or, more often, produces a parse error that surfaces as `FrontmatterUnrepresentable`. +That's the only engine instance. Bare files (no `---` block) are tolerated; +non-YAML frontmatter is either silently treated as part of the body or, more +often, produces a parse error that surfaces as `FrontmatterUnrepresentable`. ## Design ### Auto-detection by leading delimiter -A `.md` file's first non-empty line uniquely identifies the format gray_matter expects: +A `.md` file's first non-empty line uniquely identifies the format gray_matter +expects: -| First line | Format | -|---|---| -| `---` | YAML | -| `+++` | TOML | -| starts with `{` | JSON | -| anything else | bare file, no frontmatter | +| First line | Format | +| --------------- | ------------------------- | +| `---` | YAML | +| `+++` | TOML | +| starts with `{` | JSON | +| anything else | bare file, no frontmatter | -The probe is cheap (one line read) and unambiguous. No file is valid in more than one format simultaneously — these three delimiters don't collide. +The probe is cheap (one line read) and unambiguous. No file is valid in more +than one format simultaneously — these three delimiters don't collide. ### Engine dispatch in scan @@ -93,11 +104,15 @@ let pod = match detect_engine(content) { }; ``` -`gray_matter::Pod` is engine-agnostic — the downstream YAML→JSON conversion (`yaml2json` logic) needs no change for TOML/JSON since they're already trivially representable as JSON. +`gray_matter::Pod` is engine-agnostic — the downstream YAML→JSON conversion +(`yaml2json` logic) needs no change for TOML/JSON since they're already +trivially representable as JSON. ### Explicit config override -Auto-detect is the default, but some users may want to **force** a single format (e.g., to fail loudly on a `.md` file that someone wrote with the wrong delimiter). Expose this in `mdvs.toml`: +Auto-detect is the default, but some users may want to **force** a single format +(e.g., to fail loudly on a `.md` file that someone wrote with the wrong +delimiter). Expose this in `mdvs.toml`: ```toml [scan] @@ -106,28 +121,48 @@ frontmatter_format = "auto" # default; or "yaml" | "toml" | "json" Behavior: -- **`"auto"` (default)** — probe each file's first line, dispatch accordingly. Files with no recognized delimiter are bare. -- **`"yaml"` / `"toml"` / `"json"`** — skip the probe; assume every file uses that format. Files starting with a different delimiter become parse errors (`FrontmatterUnrepresentable`), which is exactly what an opinionated user would want. +- **`"auto"` (default)** — probe each file's first line, dispatch accordingly. + Files with no recognized delimiter are bare. +- **`"yaml"` / `"toml"` / `"json"`** — skip the probe; assume every file uses + that format. Files starting with a different delimiter become parse errors + (`FrontmatterUnrepresentable`), which is exactly what an opinionated user + would want. -Implementation goes in `schema::shared::ScanConfig` as a new field with serde default `"auto"`. Add a `FrontmatterFormat` enum (Auto / Yaml / Toml / Json) with `#[serde(rename_all = "kebab-case")]`. +Implementation goes in `schema::shared::ScanConfig` as a new field with serde +default `"auto"`. Add a `FrontmatterFormat` enum (Auto / Yaml / Toml / Json) +with `#[serde(rename_all = "kebab-case")]`. ### Error surface -When auto-detect fails (file starts with something that *looks* like a delimiter but the engine can't parse it — e.g., the TOML inside `+++` is malformed), we already have `FrontmatterUnrepresentable` plumbing. New errors land in the same channel; no new violation kind needed. +When auto-detect fails (file starts with something that _looks_ like a delimiter +but the engine can't parse it — e.g., the TOML inside `+++` is malformed), we +already have `FrontmatterUnrepresentable` plumbing. New errors land in the same +channel; no new violation kind needed. -When forced format mismatches (`frontmatter_format = "toml"` but a file starts with `---`), surface as a clear error mentioning the detected delimiter so the user knows what to fix. +When forced format mismatches (`frontmatter_format = "toml"` but a file starts +with `---`), surface as a clear error mentioning the detected delimiter so the +user knows what to fix. ## User-visible behavior ### Migration -Zero migration for existing users. `[scan].frontmatter_format` defaults to `"auto"` via `#[serde(default)]` — `mdvs.toml` files that don't mention the field continue to work exactly as before. No re-init, no re-build, no edits. +Zero migration for existing users. `[scan].frontmatter_format` defaults to +`"auto"` via `#[serde(default)]` — `mdvs.toml` files that don't mention the +field continue to work exactly as before. No re-init, no re-build, no edits. -The only observable behavior change for an existing vault: a `.md` file that *used* to be silently treated as a bare file because it started with `+++` or `{` (gray_matter's default `Matter::::new()` failed to parse it as YAML) will now be parsed correctly. That's strictly a fix, not a break. +The only observable behavior change for an existing vault: a `.md` file that +_used_ to be silently treated as a bare file because it started with `+++` or +`{` (gray_matter's default `Matter::::new()` failed to parse it as YAML) +will now be parsed correctly. That's strictly a fix, not a break. ### Mixed-format vaults -In Auto mode (default), per-file detection means a single vault can contain `notes-yaml.md` (`---`), `notes-toml.md` (`+++`), and `notes-json.md` (`{...}`) side by side. All three deserialize to the same JSON representation, so the schema (`[fields]` block) is format-agnostic — a `tags: Array(String)` field validates identically regardless of source format. +In Auto mode (default), per-file detection means a single vault can contain +`notes-yaml.md` (`---`), `notes-toml.md` (`+++`), and `notes-json.md` (`{...}`) +side by side. All three deserialize to the same JSON representation, so the +schema (`[fields]` block) is format-agnostic — a `tags: Array(String)` field +validates identically regardless of source format. ### Explicit format choice @@ -138,92 +173,157 @@ Optional opt-in for opinionated users: frontmatter_format = "toml" # or "yaml", "json", or "auto" (default) ``` -Forces all scanned files to use that format. Files that start with a different delimiter produce a clear `FrontmatterUnrepresentable` error naming the detected vs configured format. Useful for e.g. a Hugo site committing to TOML that wants mdvs to fail loudly when someone sneaks in a `---` file. +Forces all scanned files to use that format. Files that start with a different +delimiter produce a clear `FrontmatterUnrepresentable` error naming the detected +vs configured format. Useful for e.g. a Hugo site committing to TOML that wants +mdvs to fail loudly when someone sneaks in a `---` file. -No file-level metadata, no per-directory override — only the global `[scan]` setting (see Out of scope). +No file-level metadata, no per-directory override — only the global `[scan]` +setting (see Out of scope). ### Naming note (avoid confusion in docs) -`[scan].frontmatter_format = "toml"` controls how mdvs parses *frontmatter in `.md` files*. It has nothing to do with `mdvs.toml` itself, which is always TOML format because it's a config file. Two unrelated uses of "TOML" in the project — worth one sentence of clarification wherever this field is documented. +`[scan].frontmatter_format = "toml"` controls how mdvs parses _frontmatter in +`.md` files_. It has nothing to do with `mdvs.toml` itself, which is always TOML +format because it's a config file. Two unrelated uses of "TOML" in the project — +worth one sentence of clarification wherever this field is documented. ### Performance overhead -Negligible. Scan already hoists `Matter::::new()` outside the file loop and reuses it; the new code will pre-build three matter instances (YAML / TOML / JSON, each with the correct delimiter set) and pick one per file based on `detect_engine`'s O(1) first-line probe. +Negligible. Scan already hoists `Matter::::new()` outside the file loop +and reuses it; the new code will pre-build three matter instances (YAML / TOML / +JSON, each with the correct delimiter set) and pick one per file based on +`detect_engine`'s O(1) first-line probe. -Per-file overhead added by detection: one `content.lines().next()` call (reads at most one line, ~20 chars in practice) + two-three small string comparisons + an enum match. Order-of-magnitude: a few hundred nanoseconds per file. For a 1000-file vault, total detection overhead is sub-millisecond — dominated by 3-4 orders of magnitude by the actual frontmatter parsing we'd do regardless. +Per-file overhead added by detection: one `content.lines().next()` call (reads +at most one line, ~20 chars in practice) + two-three small string comparisons + +an enum match. Order-of-magnitude: a few hundred nanoseconds per file. For a +1000-file vault, total detection overhead is sub-millisecond — dominated by 3-4 +orders of magnitude by the actual frontmatter parsing we'd do regardless. -Forced mode (explicit `frontmatter_format = "yaml"`) skips `detect_engine` entirely — slightly faster than Auto. Zero overhead during `search` / `build` / `check` beyond scan: once frontmatter is parsed into JSON the downstream pipeline is identical across formats. +Forced mode (explicit `frontmatter_format = "yaml"`) skips `detect_engine` +entirely — slightly faster than Auto. Zero overhead during `search` / `build` / +`check` beyond scan: once frontmatter is parsed into JSON the downstream +pipeline is identical across formats. ## Open design questions -1. **What about JSON frontmatter detection precision?** *"Starts with `{`"* is loose — a markdown file could legitimately start with a JSON-looking inline code block. Tighter check: require the `{` to be the first non-whitespace character of the file AND there's a matching `}` before any normal markdown content. Should be safe to defer to `gray_matter::Matter::::new().parse()` to do the actual structural check — if it can't parse, we treat as a bare file. **Decision: rely on the parser; don't pre-validate.** - -2. **JSON edge case: how does gray_matter delimit the JSON block?** Need to verify on a real fixture. If `{ }`-delimited spans multiple lines, we need to make sure the body extraction is correct. Test fixture required. - -3. **TOML date/datetime types in frontmatter.** TOML natively has `Date` and `DateTime` types (unquoted, e.g., `joined = 2024-03-14`). `gray_matter`'s TOML engine should convert these to its Pod representation; downstream our existing Date/DateTime inference will need to handle them coming in either as strings (already-typed) or as native TOML dates. Verify on a fixture. - -4. **YAML edge case: TOML-looking comments.** A YAML file with `+++` in a string isn't going to hit this code (the first *line* check), but a TOML file with `---` in a comment ditto. No collision. Safe. - -5. **Backward compatibility.** Existing vaults written for the YAML-only mdvs continue to work because (a) auto-detect picks YAML for `---` files, and (b) the existing schema doesn't have `frontmatter_format` so serde defaults to `Auto`. No migration needed. - -6. **Should TOML frontmatter use the same `mdvs.toml` syntax?** Worth flagging that the answer is *no* — `mdvs.toml` is the schema config (TOML format because it's a config file), not frontmatter. Two unrelated uses of TOML. No conflict, but worth a sentence in docs to avoid confusion. +1. **What about JSON frontmatter detection precision?** _"Starts with `{`"_ is + loose — a markdown file could legitimately start with a JSON-looking inline + code block. Tighter check: require the `{` to be the first non-whitespace + character of the file AND there's a matching `}` before any normal markdown + content. Should be safe to defer to + `gray_matter::Matter::::new().parse()` to do the actual structural + check — if it can't parse, we treat as a bare file. **Decision: rely on the + parser; don't pre-validate.** + +2. **JSON edge case: how does gray_matter delimit the JSON block?** Need to + verify on a real fixture. If `{ }`-delimited spans multiple lines, we need to + make sure the body extraction is correct. Test fixture required. + +3. **TOML date/datetime types in frontmatter.** TOML natively has `Date` and + `DateTime` types (unquoted, e.g., `joined = 2024-03-14`). `gray_matter`'s + TOML engine should convert these to its Pod representation; downstream our + existing Date/DateTime inference will need to handle them coming in either as + strings (already-typed) or as native TOML dates. Verify on a fixture. + +4. **YAML edge case: TOML-looking comments.** A YAML file with `+++` in a string + isn't going to hit this code (the first _line_ check), but a TOML file with + `---` in a comment ditto. No collision. Safe. + +5. **Backward compatibility.** Existing vaults written for the YAML-only mdvs + continue to work because (a) auto-detect picks YAML for `---` files, and (b) + the existing schema doesn't have `frontmatter_format` so serde defaults to + `Auto`. No migration needed. + +6. **Should TOML frontmatter use the same `mdvs.toml` syntax?** Worth flagging + that the answer is _no_ — `mdvs.toml` is the schema config (TOML format + because it's a config file), not frontmatter. Two unrelated uses of TOML. No + conflict, but worth a sentence in docs to avoid confusion. ## Files to touch ### Code -- `crates/mdvs/src/discover/scan.rs` — add `FrontmatterEngine` + `detect_engine` + dispatch; replace the single `Matter::::new()` call site -- `crates/mdvs/src/schema/shared.rs` — add `FrontmatterFormat` enum + `ScanConfig.frontmatter_format: FrontmatterFormat` field with serde default -- `crates/mdvs/src/schema/config.rs` — wire the new ScanConfig field through `MdvsToml` -- (optional) `crates/mdvs/src/schema/config.rs::MdvsToml::validate()` — no new invariant needed; the FrontmatterFormat enum's serde already rejects unknown strings +- `crates/mdvs/src/discover/scan.rs` — add `FrontmatterEngine` + + `detect_engine` + dispatch; replace the single `Matter::::new()` call + site +- `crates/mdvs/src/schema/shared.rs` — add `FrontmatterFormat` enum + + `ScanConfig.frontmatter_format: FrontmatterFormat` field with serde default +- `crates/mdvs/src/schema/config.rs` — wire the new ScanConfig field through + `MdvsToml` +- (optional) `crates/mdvs/src/schema/config.rs::MdvsToml::validate()` — no new + invariant needed; the FrontmatterFormat enum's serde already rejects unknown + strings ### Tests -- New fixture: `crates/mdvs/tests/fixtures/frontmatter-toml/` with 2-3 `.md` files using `+++` TOML -- New fixture: `crates/mdvs/tests/fixtures/frontmatter-json/` with 2-3 `.md` files using JSON braces -- New fixture: `crates/mdvs/tests/fixtures/frontmatter-mixed/` with one of each — exercises auto-detect +- New fixture: `crates/mdvs/tests/fixtures/frontmatter-toml/` with 2-3 `.md` + files using `+++` TOML +- New fixture: `crates/mdvs/tests/fixtures/frontmatter-json/` with 2-3 `.md` + files using JSON braces +- New fixture: `crates/mdvs/tests/fixtures/frontmatter-mixed/` with one of each + — exercises auto-detect - Existing YAML test fixtures continue to pass unchanged -- Tests for the forced-format error case (`frontmatter_format = "toml"` + a YAML file → clear error) +- Tests for the forced-format error case (`frontmatter_format = "toml"` + a YAML + file → clear error) - Tests for malformed-TOML and malformed-JSON inside valid delimiters ### example_kb -Add 1-2 example files in `example_kb/` with TOML frontmatter to demonstrate the feature works in practice. Optional but helpful for showing off in the demo / docs. +Add 1-2 example files in `example_kb/` with TOML frontmatter to demonstrate the +feature works in practice. Optional but helpful for showing off in the demo / +docs. ### Specs -- `docs/spec/architecture.md` — data pipeline mentions YAML; broaden to "YAML / TOML / JSON" +- `docs/spec/architecture.md` — data pipeline mentions YAML; broaden to "YAML / + TOML / JSON" - `docs/spec/storage.md` — same - `docs/spec/cocogitto.md` — n/a - `AGENTS.md` — bullets mention YAML; broaden ### mdBook -- `book/src/introduction.md` — "YAML frontmatter" → "YAML, TOML, or JSON frontmatter" -- `book/src/configuration.md` — document the `[scan].frontmatter_format` field with the four allowed values and the auto behavior -- `book/src/concepts/types.md` — note TOML/JSON support; mention TOML's native Date / DateTime types -- `book/src/recipes/obsidian.md` — Obsidian uses YAML, no change beyond clarifying that other tools may use TOML/JSON +- `book/src/introduction.md` — "YAML frontmatter" → "YAML, TOML, or JSON + frontmatter" +- `book/src/configuration.md` — document the `[scan].frontmatter_format` field + with the four allowed values and the auto behavior +- `book/src/concepts/types.md` — note TOML/JSON support; mention TOML's native + Date / DateTime types +- `book/src/recipes/obsidian.md` — Obsidian uses YAML, no change beyond + clarifying that other tools may use TOML/JSON - `book/src/recipes/ci.md` — n/a (format-agnostic) -- New: `book/src/recipes/hugo.md` or similar — show how to point mdvs at a Hugo site that uses TOML +- New: `book/src/recipes/hugo.md` or similar — show how to point mdvs at a Hugo + site that uses TOML ### README -- Features list: *"Schema inference"* bullet — add "YAML, TOML, or JSON frontmatter — auto-detected per file." +- Features list: _"Schema inference"_ bullet — add "YAML, TOML, or JSON + frontmatter — auto-detected per file." - The "Why mdvs?" example uses YAML; that stays. The "Features" line is enough. ### Skill file -- `crates/mdvs/skills/mdvs/SKILL.md` — line that says "Validates frontmatter" can stay generic; the per-format detail is only relevant in setup discussion +- `crates/mdvs/skills/mdvs/SKILL.md` — line that says "Validates frontmatter" + can stay generic; the per-format detail is only relevant in setup discussion ## Work order -Implementation lives on branch `feat/multi-format-frontmatter` (status flipped from `todo` → `in-progress` 2026-05-26 at start of work). All steps land in a single PR; commits along the way use conventional types (`feat(scan):`, `test(scan):`, `docs:` per phase). +Implementation lives on branch `feat/multi-format-frontmatter` (status flipped +from `todo` → `in-progress` 2026-05-26 at start of work). All steps land in a +single PR; commits along the way use conventional types (`feat(scan):`, +`test(scan):`, `docs:` per phase). ### 1. Spike — verify gray_matter's TOML engine on native `Date` / `DateTime` ✓ **DONE 2026-05-26** -Spike committed at `scripts/test_toml_frontmatter_pod.rs`. Verifies how `gray_matter::Matter::` represents native TOML `Date` (`joined = 2024-03-14`) and `DateTime` (`synced_at = 2024-03-14T10:25:00Z`) values when round-tripped through `Pod` → `serde_json::Value`. +Spike committed at `scripts/test_toml_frontmatter_pod.rs`. Verifies how +`gray_matter::Matter::` represents native TOML `Date` +(`joined = 2024-03-14`) and `DateTime` (`synced_at = 2024-03-14T10:25:00Z`) +values when round-tripped through `Pod` → `serde_json::Value`. -**Result — Outcome A confirmed.** Both come through as plain strings in the format mdvs's existing Date / DateTime inference expects: +**Result — Outcome A confirmed.** Both come through as plain strings in the +format mdvs's existing Date / DateTime inference expects: ``` joined = "2024-03-14" (string) @@ -233,21 +333,35 @@ explicit_string_date = "2024-03-14" (string) **No Pod→JSON conversion arm needed.** Downstream inference is unchanged. -**Two implementation gotchas the spike surfaced** (didn't change the design but feed into steps 2-3): +**Two implementation gotchas the spike surfaced** (didn't change the design but +feed into steps 2-3): -- **Feature gate.** `gray_matter` default features = `["yaml"]` only. The TOML and JSON engines are gated behind `toml` and `json` features. mdvs's current `Cargo.toml` (`gray_matter = "0.3"` with no features) won't compile these in. Step 2 needs `gray_matter = { version = "0.3", features = ["toml", "json"] }`. -- **Delimiter per engine.** `Matter::new()` defaults to `delimiter = "---"` regardless of the engine. The engine choice does *not* auto-pick the delimiter; we have to set it explicitly. Step 3's dispatch needs to do: +- **Feature gate.** `gray_matter` default features = `["yaml"]` only. The TOML + and JSON engines are gated behind `toml` and `json` features. mdvs's current + `Cargo.toml` (`gray_matter = "0.3"` with no features) won't compile these in. + Step 2 needs `gray_matter = { version = "0.3", features = ["toml", "json"] }`. +- **Delimiter per engine.** `Matter::new()` defaults to `delimiter = "---"` + regardless of the engine. The engine choice does _not_ auto-pick the + delimiter; we have to set it explicitly. Step 3's dispatch needs to do: - YAML: `matter.delimiter = "---"` (default, no change needed) - TOML: `matter.delimiter = "+++"` - - JSON: `matter.delimiter = "{"` + `matter.close_delimiter = Some("}".to_string())` (verify on first JSON parse — asymmetric pair) + - JSON: `matter.delimiter = "{"` + + `matter.close_delimiter = Some("}".to_string())` (verify on first JSON parse + — asymmetric pair) ### 2. Schema config + Cargo.toml feature flags -- `crates/mdvs/Cargo.toml`: change `gray_matter = "0.3"` → `gray_matter = { version = "0.3", features = ["toml", "json"] }` (per step 1 finding — default features are yaml-only). -- `crates/mdvs/src/schema/shared.rs`: add `FrontmatterFormat` enum with `#[serde(rename_all = "kebab-case")]`. Variants `Auto`, `Yaml`, `Toml`, `Json`. `Default` impl = `Auto`. -- Add `frontmatter_format: FrontmatterFormat` field to `ScanConfig` with `#[serde(default)]`. +- `crates/mdvs/Cargo.toml`: change `gray_matter = "0.3"` → + `gray_matter = { version = "0.3", features = ["toml", "json"] }` (per step 1 + finding — default features are yaml-only). +- `crates/mdvs/src/schema/shared.rs`: add `FrontmatterFormat` enum with + `#[serde(rename_all = "kebab-case")]`. Variants `Auto`, `Yaml`, `Toml`, + `Json`. `Default` impl = `Auto`. +- Add `frontmatter_format: FrontmatterFormat` field to `ScanConfig` with + `#[serde(default)]`. - Wire through `MdvsToml` (if it explicitly composes `ScanConfig`). -- No new `MdvsToml::validate()` invariant — serde already rejects unknown values via the enum. +- No new `MdvsToml::validate()` invariant — serde already rejects unknown values + via the enum. - `cargo build` green. ### 3. Scan dispatch ✓ **DONE 2026-05-26** @@ -255,32 +369,51 @@ explicit_string_date = "2024-03-14" (string) Implemented in `crates/mdvs/src/discover/scan.rs`. Highlights: - `FrontmatterEngine` enum (internal) + `delimiter()` / `format_name()` helpers -- `detect_engine(content)` — first-non-empty-line probe (`---` / `+++` / `{` / `None`) -- `forced_engine(FrontmatterFormat)` — maps the user-facing config enum to the internal engine, or `None` for `Auto` +- `detect_engine(content)` — first-non-empty-line probe (`---` / `+++` / `{` / + `None`) +- `forced_engine(FrontmatterFormat)` — maps the user-facing config enum to the + internal engine, or `None` for `Auto` - Two parser helpers with a uniform `(EngineParse, Option)` output: - `parse_via_gray_matter(&Matter, &str)` for YAML + TOML - - `parse_json_native(&str)` for JSON (uses `serde_json::Deserializer::into_iter().byte_offset()`) -- `scan()` rewired: per-file engine resolution → forced-mode mismatch check → dispatch → unified downstream + - `parse_json_native(&str)` for JSON (uses + `serde_json::Deserializer::into_iter().byte_offset()`) +- `scan()` rewired: per-file engine resolution → forced-mode mismatch check → + dispatch → unified downstream **JSON convention decision: Hugo-style bare braces.** -The original step 1 hypothesis was that `Matter::` with `delimiter = "{"` + `close_delimiter = Some("}")` would parse Hugo-style JSON frontmatter. Verified during step 3 that **this does not work**: gray_matter strips delimiter lines and hands only the content *between* them to the engine, so the JSON engine would receive `"title": "x"` (no enclosing braces) — invalid JSON. +The original step 1 hypothesis was that `Matter::` with +`delimiter = "{"` + `close_delimiter = Some("}")` would parse Hugo-style JSON +frontmatter. Verified during step 3 that **this does not work**: `gray_matter` +strips delimiter lines and hands only the content _between_ them to the engine, +so the JSON engine would receive `"title": "x"` (no enclosing braces) — invalid +JSON. Checked the real-world conventions (web search, 2026-05-26): -- **Hugo** — bare `{...}` braces are *part of* the JSON object, on their own lines. Hugo determines format by leading delimiter. -- **Eleventy** — wraps JSON in `---json\n{...}\n---` (gray_matter language-flag convention). -- **Astro / MDX** — primarily YAML, optional TOML; JSON frontmatter inside `.md` files is not a real convention. +- **Hugo** — bare `{...}` braces are _part of_ the JSON object, on their own + lines. Hugo determines format by leading delimiter. +- **Eleventy** — wraps JSON in `---json\n{...}\n---` (gray_matter language-flag + convention). +- **Astro / MDX** — primarily YAML, optional TOML; JSON frontmatter inside `.md` + files is not a real convention. We picked **Hugo's bare-braces convention** for v0 because: -- It's unambiguous via leading-delimiter probe (no other format starts with `{`). +- It's unambiguous via leading-delimiter probe (no other format starts with + `{`). - It auto-detects cleanly alongside YAML and TOML in mixed-format vaults. -- The npm gray-matter ecosystem's `---json` convention conflicts with our YAML detection (`---` opener) and would require a language-flag probe. +- The npm gray-matter ecosystem's `---json` convention conflicts with our YAML + detection (`---` opener) and would require a language-flag probe. -Eleventy users and other custom conventions are deferred to **[TODO-0163](TODO-0163.md)** — *Expose gray_matter delimiter / engine knobs to let users customize frontmatter parsing*. +Eleventy users and other custom conventions are deferred to +**[TODO-0163](TODO-0163.md)** — _Expose gray_matter delimiter / engine knobs to +let users customize frontmatter parsing_. -**Implementation detail**: JSON does not use a `gray_matter::Matter` instance. It parses one JSON value from the start of the file via `serde_json::Deserializer::into_iter().byte_offset()`; the body is everything after the matching `}`. +**Implementation detail**: JSON does not use a `gray_matter::Matter` instance. +It parses one JSON value from the start of the file via +`serde_json::Deserializer::into_iter().byte_offset()`; the body is everything +after the matching `}`. **Per-engine delimiter rules now in code** (TOML + YAML only, JSON is custom): @@ -288,75 +421,156 @@ Eleventy users and other custom conventions are deferred to **[TODO-0163](TODO-0 - TOML: `delimiter = "+++"` (must be set explicitly per step 1 finding) - JSON: custom path via `serde_json` — no gray_matter Matter instance -**Auto / Forced modes**: Auto uses `detect_engine` and treats `None` as bare (preserves existing behavior). Forced modes detect mismatches up front and surface a clear `FrontmatterUnrepresentable` with both configured and detected delimiters named. +**Auto / Forced modes**: Auto uses `detect_engine` and treats `None` as bare +(preserves existing behavior). Forced modes detect mismatches up front and +surface a clear `FrontmatterUnrepresentable` with both configured and detected +delimiters named. -**Tests added** (all pass, 36 scan tests total): unit tests for `detect_engine` (each delimiter, bare, empty, leading-blank-lines, trailing-whitespace); integration tests for TOML parsing, TOML native `Date` round-trip, JSON parsing, mixed-format vault auto-dispatch, forced-mode mismatches (yaml→toml, toml→yaml, json→yaml), and forced-mode accept paths. +**Tests added** (all pass, 36 scan tests total): unit tests for `detect_engine` +(each delimiter, bare, empty, leading-blank-lines, trailing-whitespace); +integration tests for TOML parsing, TOML native `Date` round-trip, JSON parsing, +mixed-format vault auto-dispatch, forced-mode mismatches (yaml→toml, toml→yaml, +json→yaml), and forced-mode accept paths. ### 4. Test fixtures ✓ **DONE 2026-05-26** Created 9 fixture files under `crates/mdvs/tests/fixtures/`: -- `frontmatter-toml/` (3 files) — `book.md` (basic scalars + array), `release.md` (native TOML `Date` + `DateTime` via unquoted `2026-05-26` / `2026-05-26T14:30:00Z`), `nested.md` (TOML native nested-table `[calibration.baseline]` syntax → dotted-name leaves) -- `frontmatter-json/` (3 files) — `book.md`, `release.md` (RFC 3339 strings for dates), `nested.md` (nested objects → dotted-name leaves) -- `frontmatter-mixed/` (3 files) — `yaml-note.md`, `toml-note.md`, `json-note.md` with identical schemas in different formats, exercising auto-detect across one vault +- `frontmatter-toml/` (3 files) — `book.md` (basic scalars + array), + `release.md` (native TOML `Date` + `DateTime` via unquoted `2026-05-26` / + `2026-05-26T14:30:00Z`), `nested.md` (TOML native nested-table + `[calibration.baseline]` syntax → dotted-name leaves) +- `frontmatter-json/` (3 files) — `book.md`, `release.md` (RFC 3339 strings for + dates), `nested.md` (nested objects → dotted-name leaves) +- `frontmatter-mixed/` (3 files) — `yaml-note.md`, `toml-note.md`, + `json-note.md` with identical schemas in different formats, exercising + auto-detect across one vault Smoke-tested via `mdvs init` on each fixture: -- TOML vault: 13 fields inferred including `built_at: DateTime`, `released_on: Date`, dotted `calibration.baseline.*: Float`. -- JSON vault: same 13 fields, with `built_at: DateTime` and `released_on: Date` promoted from RFC 3339 strings. -- Mixed vault: 4 unified fields (`title`, `author`, `year`, `tags`) each present in all 3 files — proves YAML / TOML / JSON deserialize to identical JSON shape. +- TOML vault: 13 fields inferred including `built_at: DateTime`, + `released_on: Date`, dotted `calibration.baseline.*: Float`. +- JSON vault: same 13 fields, with `built_at: DateTime` and `released_on: Date` + promoted from RFC 3339 strings. +- Mixed vault: 4 unified fields (`title`, `author`, `year`, `tags`) each present + in all 3 files — proves YAML / TOML / JSON deserialize to identical JSON + shape. -Existing scan unit tests in `crates/mdvs/src/discover/scan.rs` already cover YAML behavior; no separate YAML fixture directory needed. +Existing scan unit tests in `crates/mdvs/src/discover/scan.rs` already cover +YAML behavior; no separate YAML fixture directory needed. ### 5. Tests ✓ **DONE 2026-05-26** -Step 3 already covered the unit-level surface (detect_engine + forced-mode mismatch + per-engine parse). Step 5 added 8 integration tests in `crates/mdvs/tests/multi_format_frontmatter.rs` that exercise the command-level pipeline (`init` → reload → `check`) against the fixture vaults: - -- `init_infers_toml_vault` — TOML native Date / DateTime → typed fields; `[calibration.baseline]` → dotted-name leaves -- `init_infers_json_vault` — RFC 3339 strings → Date / DateTime; nested JSON → dotted leaves -- `init_infers_mixed_vault` — 3 files in 3 formats unify into 4-field schema, all required +Step 3 already covered the unit-level surface (detect_engine + forced-mode +mismatch + per-engine parse). Step 5 added 8 integration tests in +`crates/mdvs/tests/multi_format_frontmatter.rs` that exercise the command-level +pipeline (`init` → reload → `check`) against the fixture vaults: + +- `init_infers_toml_vault` — TOML native Date / DateTime → typed fields; + `[calibration.baseline]` → dotted-name leaves +- `init_infers_json_vault` — RFC 3339 strings → Date / DateTime; nested JSON → + dotted leaves +- `init_infers_mixed_vault` — 3 files in 3 formats unify into 4-field schema, + all required - `check_passes_on_{toml,json,mixed}_vault` — zero violations after init -- `malformed_{toml,json}_surfaces_violation` — broken content inside valid delimiters produces `FrontmatterUnrepresentable` - -**Side effect of the malformed tests** — `parse_via_gray_matter` previously swallowed gray_matter parse errors and treated the file as bare (legacy YAML-only behavior). Now it surfaces them as `FrontmatterUnrepresentable` with the engine's error message attached. This unifies behavior across YAML / TOML / JSON: broken frontmatter is always a violation, regardless of format. Strict improvement over the silent-fallback path; no existing test broke. - -**Verification:** `cargo test -p mdvs` → 806 unit tests + 8 integration tests pass, no regressions. `cargo clippy --all-targets` clean. `cargo fmt` applied. - -**Skipped for now (rationale):** build + search end-to-end tests through each fixture would require loading the `model2vec` model (~30 MB from HuggingFace cache), slowing the test suite materially for redundant coverage — `check` already proves the new dispatch produces valid downstream Values, and build/search consume those Values via the same path YAML already tests. Will revisit in step 8 (verification) if a smoke test is wanted. +- `malformed_{toml,json}_surfaces_violation` — broken content inside valid + delimiters produces `FrontmatterUnrepresentable` + +**Side effect of the malformed tests** — `parse_via_gray_matter` previously +swallowed gray_matter parse errors and treated the file as bare (legacy +YAML-only behavior). Now it surfaces them as `FrontmatterUnrepresentable` with +the engine's error message attached. This unifies behavior across YAML / TOML / +JSON: broken frontmatter is always a violation, regardless of format. Strict +improvement over the silent-fallback path; no existing test broke. + +**Verification:** `cargo test -p mdvs` → 806 unit tests + 8 integration tests +pass, no regressions. `cargo clippy --all-targets` clean. `cargo fmt` applied. + +**Skipped for now (rationale):** build + search end-to-end tests through each +fixture would require loading the `model2vec` model (~30 MB from HuggingFace +cache), slowing the test suite materially for redundant coverage — `check` +already proves the new dispatch produces valid downstream Values, and +build/search consume those Values via the same path YAML already tests. Will +revisit in step 8 (verification) if a smoke test is wanted. ### 6. example_kb dogfood ✓ **DONE 2026-05-26** -Resolved: **add new files in TOML and JSON formats alongside the existing YAML notes** (Plan A variant — additive only, no replacement of existing files). Two new files committed: - -- `example_kb/reference/protocols/equipment/spec-200x-calibration-log.md` — TOML frontmatter for a monthly SPEC-200X calibration log. Exercises native TOML `Date` (`last_reviewed = 2032-02-14`) and `DateTime` (`synced_at = 2032-02-14T09:14:32Z`) literals. Body explains in-universe that the calibration vendor utility exports TOML which the lab journal appends verbatim. -- `example_kb/projects/alpha/notes/experiment-4.md` — JSON frontmatter for REMO's A-046 humidity follow-up sweep. Hugo-style bare braces. Body explains in-universe that REMO's automation pipeline serializes experiment metadata as JSON for downstream tooling. - -Both files **only use existing field names** to avoid schema churn. Two `allowed` globs in `mdvs.toml` were widened by one path entry each (`firmware_version` to include `reference/protocols/equipment/*`, `synced_at` to include the same) — these are manual constraint adjustments that `mdvs update` doesn't perform; they were the minimum touch to keep `mdvs check example_kb` returning zero violations. +Resolved: **add new files in TOML and JSON formats alongside the existing YAML +notes** (Plan A variant — additive only, no replacement of existing files). Two +new files committed: + +- `example_kb/reference/protocols/equipment/spec-200x-calibration-log.md` — TOML + frontmatter for a monthly SPEC-200X calibration log. Exercises native TOML + `Date` (`last_reviewed = 2032-02-14`) and `DateTime` + (`synced_at = 2032-02-14T09:14:32Z`) literals. Body explains in-universe that + the calibration vendor utility exports TOML which the lab journal appends + verbatim. +- `example_kb/projects/alpha/notes/experiment-4.md` — JSON frontmatter for + REMO's A-046 humidity follow-up sweep. Hugo-style bare braces. Body explains + in-universe that REMO's automation pipeline serializes experiment metadata as + JSON for downstream tooling. + +Both files **only use existing field names** to avoid schema churn. Two +`allowed` globs in `mdvs.toml` were widened by one path entry each +(`firmware_version` to include `reference/protocols/equipment/*`, `synced_at` to +include the same) — these are manual constraint adjustments that `mdvs update` +doesn't perform; they were the minimum touch to keep `mdvs check example_kb` +returning zero violations. Verification: `mdvs check example_kb` → 45 files, zero violations. ### 7. Doc sweep ✓ **DONE 2026-05-26** -User-visible docs (mdbook stays YAML-first; multi-format is surfaced as an extended capability in the dedicated places): - -- `README.md` — frontmatter intro reworded to introduce all three formats with auto-detection; added a new "**Multi-format frontmatter**" Features bullet (marketing surface — multi-format is a launch differentiator and should be visible) -- `book/src/introduction.md` — kept YAML as the primary example; appended a one-line callout pointing readers to the configuration field and the Hugo recipe -- `book/src/configuration.md` — `[scan]` table extended with `frontmatter_format`; new "Frontmatter format" subsection covers Auto vs forced modes and the naming-clash caveat (`mdvs.toml` is config, frontmatter `.toml` is content). This is the reference page; full details belong here. -- `book/src/concepts/types.md` — left untouched. The page is about *types*, not about *formats*; cluttering it with multi-format prose would distract from its job -- `book/src/recipes.md` — **new parent page** for the Recipes chapter, modeled on `commands.md`: brief intro listing and linking to the three sub-pages (Obsidian / Hugo / CI). Recipes now mirrors the Commands structure exactly. -- `book/src/recipes/hugo.md` — **new recipe** covering mixed-format Hugo sites, forced-format mode, native TOML date queries, useful `--where` examples, and CI integration. -- `book/src/recipes/ci.md` — **rewritten from TBD stub into real content**: minimal GitHub Actions workflow, version-pinning rationale, `--no-update` reasoning for deterministic CI, what `check` covers vs. what it doesn't, notes on GitLab / CircleCI / pre-commit shape. -- `book/src/recipes/obsidian.md` — kept; one-line addition acknowledging multi-format support for imported notes. -- `book/src/SUMMARY.md` — Recipes moved from Reference into Guide (recipes are procedural how-tos, not lookup material). Nested structure: `Recipes` parent page with `Obsidian` / `Hugo` / `CI` as numbered sub-entries (4.1 / 4.2 / 4.3), parallel to Commands' 3.1–3.8. +User-visible docs (mdbook stays YAML-first; multi-format is surfaced as an +extended capability in the dedicated places): + +- `README.md` — frontmatter intro reworded to introduce all three formats with + auto-detection; added a new "**Multi-format frontmatter**" Features bullet + (marketing surface — multi-format is a launch differentiator and should be + visible) +- `book/src/introduction.md` — kept YAML as the primary example; appended a + one-line callout pointing readers to the configuration field and the Hugo + recipe +- `book/src/configuration.md` — `[scan]` table extended with + `frontmatter_format`; new "Frontmatter format" subsection covers Auto vs + forced modes and the naming-clash caveat (`mdvs.toml` is config, frontmatter + `.toml` is content). This is the reference page; full details belong here. +- `book/src/concepts/types.md` — left untouched. The page is about _types_, not + about _formats_; cluttering it with multi-format prose would distract from its + job +- `book/src/recipes.md` — **new parent page** for the Recipes chapter, modeled + on `commands.md`: brief intro listing and linking to the three sub-pages + (Obsidian / Hugo / CI). Recipes now mirrors the Commands structure exactly. +- `book/src/recipes/hugo.md` — **new recipe** covering mixed-format Hugo sites, + forced-format mode, native TOML date queries, useful `--where` examples, and + CI integration. +- `book/src/recipes/ci.md` — **rewritten from TBD stub into real content**: + minimal GitHub Actions workflow, version-pinning rationale, `--no-update` + reasoning for deterministic CI, what `check` covers vs. what it doesn't, notes + on GitLab / CircleCI / pre-commit shape. +- `book/src/recipes/obsidian.md` — kept; one-line addition acknowledging + multi-format support for imported notes. +- `book/src/SUMMARY.md` — Recipes moved from Reference into Guide (recipes are + procedural how-tos, not lookup material). Nested structure: `Recipes` parent + page with `Obsidian` / `Hugo` / `CI` as numbered sub-entries (4.1 / 4.2 / + 4.3), parallel to Commands' 3.1–3.8. Agent / internal docs (accuracy): -- `AGENTS.md` (symlinked from `CLAUDE.md`) — module map and data-pipeline lines broadened; new "Multi-format frontmatter (TODO-0162)" design-decision bullet; deps table reflects the JSON-via-`serde_json` split -- `crates/mdvs/skills/mdvs/SKILL.md` — one-paragraph addition under the title summarizing format support + `[scan].frontmatter_format` knob -- `docs/spec/architecture.md` — Scan stage and module map reflect per-file dispatch; "Literal-dot YAML keys" section renamed to "Literal-dot frontmatter keys" and broadened; deps table updated -- `docs/spec/storage.md` — `data` Struct column description broadened beyond YAML - -**Skipped:** `book/src/recipes/ci.md` (format-agnostic, no claim to update). `book/src/concepts/validation.md` / `concepts/search.md` (no YAML-specific claims). `docs/spec/commands/*.md` (CLI surface, not parsing). +- `AGENTS.md` (symlinked from `CLAUDE.md`) — module map and data-pipeline lines + broadened; new "Multi-format frontmatter (TODO-0162)" design-decision bullet; + deps table reflects the JSON-via-`serde_json` split +- `crates/mdvs/skills/mdvs/SKILL.md` — one-paragraph addition under the title + summarizing format support + `[scan].frontmatter_format` knob +- `docs/spec/architecture.md` — Scan stage and module map reflect per-file + dispatch; "Literal-dot YAML keys" section renamed to "Literal-dot frontmatter + keys" and broadened; deps table updated +- `docs/spec/storage.md` — `data` Struct column description broadened beyond + YAML + +**Skipped:** `book/src/recipes/ci.md` (format-agnostic, no claim to update). +`book/src/concepts/validation.md` / `concepts/search.md` (no YAML-specific +claims). `docs/spec/commands/*.md` (CLI surface, not parsing). ### 8. Verification ✓ **DONE 2026-05-26** @@ -367,31 +581,49 @@ All automated checks green: - `cargo fmt --check` → clean - `mdbook build book` → clean, no warnings -Manual smoke on `example_kb` (which carries the 2 new multi-format files from step 6): +Manual smoke on `example_kb` (which carries the 2 new multi-format files from +step 6): - `mdvs check example_kb` → 45 files, zero violations -- `mdvs build --force example_kb` → 45 files, 62 chunks, full rebuild successful (schema-hash gate fired correctly on the widened `allowed` globs from step 6 — `--force` cleared) -- `mdvs search "humidity sensitivity follow-up" example_kb` → new JSON-frontmatter `experiment-4.md` returned as **#1 hit**, confirming the full pipeline (parse → schema → embed → store → retrieve) works for JSON content end-to-end -- `mdvs search "monthly calibration log lamp hours" example_kb` → new TOML-frontmatter `spec-200x-calibration-log.md` returned as **#1 hit**, same proof for TOML - -Skipped from the original TODO: manual smoke on each of the three test fixtures under `tests/fixtures/`. Redundant — the integration tests cover `init` + `check` on those, and `example_kb` covers the build + search smoke on multi-format content in a realistic vault. +- `mdvs build --force example_kb` → 45 files, 62 chunks, full rebuild successful + (schema-hash gate fired correctly on the widened `allowed` globs from step 6 — + `--force` cleared) +- `mdvs search "humidity sensitivity follow-up" example_kb` → new + JSON-frontmatter `experiment-4.md` returned as **#1 hit**, confirming the full + pipeline (parse → schema → embed → store → retrieve) works for JSON content + end-to-end +- `mdvs search "monthly calibration log lamp hours" example_kb` → new + TOML-frontmatter `spec-200x-calibration-log.md` returned as **#1 hit**, same + proof for TOML + +Skipped from the original TODO: manual smoke on each of the three test fixtures +under `tests/fixtures/`. Redundant — the integration tests cover `init` + +`check` on those, and `example_kb` covers the build + search smoke on +multi-format content in a realistic vault. ### PR -Branch ready for PR off `feat/multi-format-frontmatter` against `main`. Nine commits, conventional-commit history per step. +Branch ready for PR off `feat/multi-format-frontmatter` against `main`. Nine +commits, conventional-commit history per step. ### PR shape -Single PR off `feat/multi-format-frontmatter` when steps 2-8 land. Conventional-commit history per phase; no per-step PR splitting (the changes are too tightly related — splitting would just create noise). +Single PR off `feat/multi-format-frontmatter` when steps 2-8 land. +Conventional-commit history per phase; no per-step PR splitting (the changes are +too tightly related — splitting would just create noise). ## Acceptance criteria -- [ ] `mdvs init` / `check` / `build` / `search` work on a vault of `.md` files with `+++`-delimited TOML frontmatter +- [ ] `mdvs init` / `check` / `build` / `search` work on a vault of `.md` files + with `+++`-delimited TOML frontmatter - [ ] Same for JSON frontmatter (`{...}` delimited) -- [ ] Mixed vault with YAML + TOML + JSON files in different directories validates correctly +- [ ] Mixed vault with YAML + TOML + JSON files in different directories + validates correctly - [ ] All existing YAML-only tests pass unchanged -- [ ] `[scan].frontmatter_format = "yaml"` (forced) raises a clear error on a `+++` file naming the detected delimiter -- [ ] TOML native `Date` / `DateTime` types convert correctly through the inference pipeline (typed Date / DateTime in `mdvs.toml`) +- [ ] `[scan].frontmatter_format = "yaml"` (forced) raises a clear error on a + `+++` file naming the detected delimiter +- [ ] TOML native `Date` / `DateTime` types convert correctly through the + inference pipeline (typed Date / DateTime in `mdvs.toml`) - [ ] `mdbook build book` clean - [ ] `cargo test -p mdvs` green - [ ] `cargo clippy --all-targets` clean @@ -399,7 +631,8 @@ Single PR off `feat/multi-format-frontmatter` when steps 2-8 land. Conventional- ## Out of scope - Custom delimiter syntax (e.g., `;;;` from some less-common tools) -- Frontmatter in formats other than the three `gray_matter` engines (YAML / TOML / JSON) +- Frontmatter in formats other than the three `gray_matter` engines (YAML / TOML + / JSON) - Per-directory `frontmatter_format` overrides — `[scan]` is global - Auto-converting between formats (e.g., "rewrite all YAML files to TOML") - `mdvs export` to a specific format — read-only is the contract diff --git a/docs/spec/todos/TODO-0163.md b/docs/spec/todos/TODO-0163.md index b564f4b..71b4621 100644 --- a/docs/spec/todos/TODO-0163.md +++ b/docs/spec/todos/TODO-0163.md @@ -12,14 +12,24 @@ blocks: [] ## Summary -TODO-0162 hard-codes three frontmatter conventions: YAML between `---` (gray_matter default), TOML between `+++` (gray_matter with custom delimiter), and Hugo-style bare-braces JSON (parsed via `serde_json` directly, bypassing gray_matter's `Matter::`). This covers the common cases but leaves out users with non-standard setups — most notably **Eleventy-style `---json` language-flag wrappers**, **custom delimiter strings** (`~~~`, ``, `;;;`), and **JavaScript front matter** (gray_matter supports it via a separate engine). +TODO-0162 hard-codes three frontmatter conventions: YAML between `---` +(gray_matter default), TOML between `+++` (gray_matter with custom delimiter), +and Hugo-style bare-braces JSON (parsed via `serde_json` directly, bypassing +gray_matter's `Matter::`). This covers the common cases but leaves out +users with non-standard setups — most notably **Eleventy-style `---json` +language-flag wrappers**, **custom delimiter strings** (`~~~`, ``, +`;;;`), and **JavaScript front matter** (gray_matter supports it via a separate +engine). + +This TODO exposes the underlying gray_matter knobs through `mdvs.toml` so an +advanced user can: -This TODO exposes the underlying gray_matter knobs through `mdvs.toml` so an advanced user can: - Override delimiters per format (e.g. `~~~` instead of `---` for YAML) - Plug in the `---json` language-flag convention (Eleventy / npm gray-matter) - Opt into custom delimiter pairs (asymmetric `open` / `close`) -Low priority — TODO-0162's hard-coded set covers the dominant ecosystems (Obsidian, Hugo, Astro). Open this when a user actually asks. +Low priority — TODO-0162's hard-coded set covers the dominant ecosystems +(Obsidian, Hugo, Astro). Open this when a user actually asks. ## Background — what's in scope @@ -34,15 +44,25 @@ pub struct Matter { } ``` -TODO-0162's implementation sets `delimiter = "+++"` for the TOML matter and leaves the YAML matter at its `"---"` default. JSON bypasses gray_matter entirely. None of these knobs are user-tunable today — they're hard-coded in `scan.rs::scan()`. +TODO-0162's implementation sets `delimiter = "+++"` for the TOML matter and +leaves the YAML matter at its `"---"` default. JSON bypasses gray_matter +entirely. None of these knobs are user-tunable today — they're hard-coded in +`scan.rs::scan()`. ## Why this is its own TODO (not part of 0162) Three reasons: -1. **Conventions aren't uniform.** Hugo uses bare `{...}` for JSON; Eleventy uses `---json\n{...}\n---`; Astro/MDX mostly skip JSON entirely. Picking one is necessary for v0; exposing both via config is a feature, not a default. -2. **The config surface is non-trivial.** Per-format delimiter + close-delimiter + language-flag would balloon `[scan]` significantly. Best designed once we know what real users actually want, rather than speculatively. -3. **Demand is unclear.** Most users sit inside one ecosystem (Obsidian → YAML, Hugo → any, Eleventy → `---json`). The hard-coded set in 0162 covers Hugo + Obsidian completely; Eleventy is the visible gap. +1. **Conventions aren't uniform.** Hugo uses bare `{...}` for JSON; Eleventy + uses `---json\n{...}\n---`; Astro/MDX mostly skip JSON entirely. Picking one + is necessary for v0; exposing both via config is a feature, not a default. +2. **The config surface is non-trivial.** Per-format delimiter + + close-delimiter + language-flag would balloon `[scan]` significantly. Best + designed once we know what real users actually want, rather than + speculatively. +3. **Demand is unclear.** Most users sit inside one ecosystem (Obsidian → YAML, + Hugo → any, Eleventy → `---json`). The hard-coded set in 0162 covers Hugo + + Obsidian completely; Eleventy is the visible gap. ## Design sketch (deferred — for discussion when we pick this up) @@ -65,7 +85,8 @@ style = "hugo" # language_tag = "json" ``` -`style` selects the parsing strategy; the other fields parameterize it. Default values match the hard-coded set TODO-0162 ships. +`style` selects the parsing strategy; the other fields parameterize it. Default +values match the hard-coded set TODO-0162 ships. ### Option B: single inline table per format @@ -83,16 +104,24 @@ More compact, slightly more awkward to extend. Probably worse for documentation. ### Custom engines via plugins -Out of scope here, but worth flagging: gray_matter has a `JS` engine (executes JavaScript front matter) and supports custom `Engine` impls. Exposing those would require a plugin system, which is bigger than just config. Not in this TODO. +Out of scope here, but worth flagging: gray_matter has a `JS` engine (executes +JavaScript front matter) and supports custom `Engine` impls. Exposing those +would require a plugin system, which is bigger than just config. Not in this +TODO. ## Acceptance criteria (when picked up) -- [ ] Users can override the YAML / TOML delimiter via `mdvs.toml` (`~~~`, ``, etc.) -- [ ] Users can switch JSON between Hugo-style (bare braces) and Eleventy-style (`---json` language flag) via config +- [ ] Users can override the YAML / TOML delimiter via `mdvs.toml` (`~~~`, + ``, etc.) +- [ ] Users can switch JSON between Hugo-style (bare braces) and Eleventy-style + (`---json` language flag) via config - [ ] Defaults are byte-identical to TODO-0162's hard-coded set (zero-migration) -- [ ] Forced-format mismatch errors mention the user's custom delimiters, not the defaults -- [ ] Tests cover at least: `~~~` YAML, `` YAML, Eleventy-style `---json` JSON -- [ ] Docs: short `book/src/recipes/custom-frontmatter.md` showing the three example overrides +- [ ] Forced-format mismatch errors mention the user's custom delimiters, not + the defaults +- [ ] Tests cover at least: `~~~` YAML, `` YAML, Eleventy-style + `---json` JSON +- [ ] Docs: short `book/src/recipes/custom-frontmatter.md` showing the three + example overrides ## Out of scope @@ -103,6 +132,8 @@ Out of scope here, but worth flagging: gray_matter has a `JS` engine (executes J ## Files to touch (when picked up) -- `crates/mdvs/src/schema/shared.rs` — extend `ScanConfig` with the override block -- `crates/mdvs/src/discover/scan.rs` — feed user overrides into the `Matter` constructors; switch JSON parser based on `style` +- `crates/mdvs/src/schema/shared.rs` — extend `ScanConfig` with the override + block +- `crates/mdvs/src/discover/scan.rs` — feed user overrides into the `Matter` + constructors; switch JSON parser based on `style` - Tests + recipe doc as above diff --git a/docs/spec/todos/TODO-0164.md b/docs/spec/todos/TODO-0164.md index eb87409..e56c374 100644 --- a/docs/spec/todos/TODO-0164.md +++ b/docs/spec/todos/TODO-0164.md @@ -13,36 +13,34 @@ blocks: [] ## Summary Today's `Constraints` struct has no way to bound the **length** of an array -field. `min_length` / `max_length` apply per-element (they target the -`items` subschema, not the array itself); `min` / `max` apply per-element -for numeric arrays. There's no knob for "this array must have at least N -elements" or "at most N elements" — a common validation need for things -like `attendees` (at least 1), `tags` (at most 10), `coordinates` (exactly -2), etc. +field. `min_length` / `max_length` apply per-element (they target the `items` +subschema, not the array itself); `min` / `max` apply per-element for numeric +arrays. There's no knob for "this array must have at least N elements" or "at +most N elements" — a common validation need for things like `attendees` (at +least 1), `tags` (at most 10), `coordinates` (exactly 2), etc. The runtime is **half-wired** already: - `minItems` / `maxItems` are in the JSON Schema allow-list (`schema/json_schema.rs:44-45`) -- The error mapper translates `MinItems` / `MaxItems` errors into - `OutOfRange` violations with rule `"minItems N"` / `"maxItems N"` - (`cmd/check.rs:809,814`) -- Round-trip tests confirm validation works when these keywords are - present in the schema (`cmd/check.rs:2385-2396`) +- The error mapper translates `MinItems` / `MaxItems` errors into `OutOfRange` + violations with rule `"minItems N"` / `"maxItems N"` (`cmd/check.rs:809,814`) +- Round-trip tests confirm validation works when these keywords are present in + the schema (`cmd/check.rs:2385-2396`) But the DSL side is empty: - `Constraints` has no `min_items` / `max_items` field - `dsl_to_canonical` never emits them - `canonical_to_dsl` would fail to round-trip them - (`mdvs init --from-jsonschema` of a schema using `minItems` would - silently drop the constraint) + (`mdvs init --from-jsonschema` of a schema using `minItems` would silently + drop the constraint) - Inference can't recommend them -So a user who imports a schema via `--from-jsonschema` containing -`minItems` / `maxItems` would get the validation behavior but couldn't -re-export it via `mdvs export-jsonschema` — a one-way trip. And users -writing `mdvs.toml` directly have no way to express the constraint at all. +So a user who imports a schema via `--from-jsonschema` containing `minItems` / +`maxItems` would get the validation behavior but couldn't re-export it via +`mdvs export-jsonschema` — a one-way trip. And users writing `mdvs.toml` +directly have no way to express the constraint at all. ## Design @@ -60,9 +58,9 @@ pub min_items: Option, pub max_items: Option, ``` -Both fields are independent of `min_length` / `max_length`. The latter -target per-element string length (`items.minLength` in the emitted -schema); the new ones target the array itself. +Both fields are independent of `min_length` / `max_length`. The latter target +per-element string length (`items.minLength` in the emitted schema); the new +ones target the array itself. ### New ConstraintKind variant @@ -80,8 +78,8 @@ pub enum ConstraintKind { `crates/mdvs/src/schema/constraints/item_count.rs`: -- `validate_for_type` — applicable to **any** `FieldType::Array(_)`, - rejected for all scalar / Object types +- `validate_for_type` — applicable to **any** `FieldType::Array(_)`, rejected + for all scalar / Object types - Sanity: if both set, `min_items <= max_items` - Both `None` is rejected (must have at least one) @@ -91,27 +89,27 @@ Pattern is exactly parallel to `length.rs`. `apply_constraints` in `json_schema.rs` already runs once on the array's property schema (before `items` is filled in by the Array branch in -`type_subschema`). But today **all** constraint emission happens -unconditionally on the inner type and `items` ends up carrying everything. +`type_subschema`). But today **all** constraint emission happens unconditionally +on the inner type and `items` ends up carrying everything. Fix: split into two emission points. - **Per-element constraints** (`categories`, `min`, `max`, `min_length`, `max_length`, `pattern`) → go on `items` (current behavior). -- **Array-level constraints** (`min_items`, `max_items`) → go on the - array property itself. +- **Array-level constraints** (`min_items`, `max_items`) → go on the array + property itself. -Concretely, the Array(*) branch in `type_subschema` should: +Concretely, the Array(\*) branch in `type_subschema` should: 1. Call `type_subschema(inner, false, constraints)` to build `items` — per-element constraints flow in there as today 2. Build the array property schema (`{"type": "array", "items": ...}`) -3. Call a new helper `apply_array_constraints(obj, constraints)` that - writes `minItems` / `maxItems` onto the array property +3. Call a new helper `apply_array_constraints(obj, constraints)` that writes + `minItems` / `maxItems` onto the array property -For scalar fields, the new helper is a no-op (or skipped). The cleanest -way is to make `apply_constraints` split by keyword: per-element keywords -go on the current target, array-level keywords go on `obj`. +For scalar fields, the new helper is a no-op (or skipped). The cleanest way is +to make `apply_constraints` split by keyword: per-element keywords go on the +current target, array-level keywords go on `obj`. ### `canonical_to_dsl` reverse @@ -126,88 +124,85 @@ if let Some(v) = src.get("maxItems").and_then(Value::as_u64) { } ``` -Sourced from the **array property level**, not the `items` level -(unlike `min_length` etc). +Sourced from the **array property level**, not the `items` level (unlike +`min_length` etc). ### MdvsToml::validate -Update invariant 4 (constraints valid for type) — when the field is -`Array(*)` and `min_items`/`max_items` are set, dispatch to -`item_count::validate_for_type`. The categories-mutual-exclusion check -needs to be re-examined: should `categories` + `min_items` coexist? -Yes — they validate orthogonal things (each element belongs to enum AND -the array has at least N elements). Keep `categories` mutually exclusive -only with the per-element overlap set (range, length, pattern). +Update invariant 4 (constraints valid for type) — when the field is `Array(*)` +and `min_items`/`max_items` are set, dispatch to +`item_count::validate_for_type`. The categories-mutual-exclusion check needs to +be re-examined: should `categories` + `min_items` coexist? Yes — they validate +orthogonal things (each element belongs to enum AND the array has at least N +elements). Keep `categories` mutually exclusive only with the per-element +overlap set (range, length, pattern). ### Inference **Not auto-inferred in v0.** There's no clean signal: seeing arrays with -`len >= 3` across all observed files doesn't mean the user wants -`min_items = 3` — it might just be coincidence. User-declared only. +`len >= 3` across all observed files doesn't mean the user wants `min_items = 3` +— it might just be coincidence. User-declared only. -(Future: could compute observed_min_array_len / observed_max_array_len -and suggest them with `# inferred from observation` comments — but -that's a separate, optional UX TODO.) +(Future: could compute observed_min_array_len / observed_max_array_len and +suggest them with `# inferred from observation` comments — but that's a +separate, optional UX TODO.) ### Storage -Lance encoding doesn't change. The constraint is purely a check-time -concern; the column type (`List<...>`) is unchanged. +Lance encoding doesn't change. The constraint is purely a check-time concern; +the column type (`List<...>`) is unchanged. ### Required vs min_items distinction -Worth noting in the docs: `min_items = 1` is **not** the same as -`required` for the field. `required` controls whether the **key** must -be present in the frontmatter; `min_items = 1` controls whether the -array (once present) must be non-empty. A nullable `Array(String)` with -`required` paths and `min_items = 1` would require: the key is present, -the value is either null OR a non-empty array. +Worth noting in the docs: `min_items = 1` is **not** the same as `required` for +the field. `required` controls whether the **key** must be present in the +frontmatter; `min_items = 1` controls whether the array (once present) must be +non-empty. A nullable `Array(String)` with `required` paths and `min_items = 1` +would require: the key is present, the value is either null OR a non-empty +array. ## Files to touch - `crates/mdvs/src/schema/constraints/mod.rs` — add fields, extend - `ConstraintKind`, update `kinds()` / `validate_config` / - `applicable_keywords` + `ConstraintKind`, update `kinds()` / `validate_config` / `applicable_keywords` - `crates/mdvs/src/schema/constraints/item_count.rs` — new module - `crates/mdvs/src/schema/json_schema.rs` — split constraint emission (per-element vs array-level), update `canonical_to_dsl` reader -- `crates/mdvs/src/cmd/check.rs` — error mapping already exists; verify - the rule strings match what users expect +- `crates/mdvs/src/cmd/check.rs` — error mapping already exists; verify the rule + strings match what users expect - `crates/mdvs/src/schema/config.rs` — `MdvsToml::validate` invariant 4 -- Tests at every layer (constraint applicability, schema emission, - round-trip, end-to-end check violation) +- Tests at every layer (constraint applicability, schema emission, round-trip, + end-to-end check violation) - `book/src/concepts/constraints.md` — document the new constraint -- `book/src/configuration.md` — mention `min_items` / `max_items` in - the constraints table +- `book/src/configuration.md` — mention `min_items` / `max_items` in the + constraints table - `docs/spec/architecture.md` — add to the constraints summary -- `docs/spec/shared.md` — note the existence (if it lists constraint - kinds) +- `docs/spec/shared.md` — note the existence (if it lists constraint kinds) ## Acceptance criteria - [ ] `mdvs.toml` accepts `min_items` and `max_items` under - `[fields.field.constraints]` for any `Array(*)` field + `[fields.field.constraints]` for any `Array(*)` field - [ ] Non-Array field types reject the constraint at config load - [ ] `min_items > max_items` is rejected at config load -- [ ] Validation fires with `OutOfRange` violation containing - `minItems N` / `maxItems N` rule when the bound is violated +- [ ] Validation fires with `OutOfRange` violation containing `minItems N` / + `maxItems N` rule when the bound is violated - [ ] `dsl_to_canonical` ↔ `canonical_to_dsl` round-trip preserves the - constraint -- [ ] `mdvs export-jsonschema` emits `minItems` / `maxItems` at the - array property level -- [ ] `mdvs init --from-jsonschema` of a schema with `minItems` / - `maxItems` produces a config that re-exports identically -- [ ] Coexistence with per-element constraints works - (e.g. `Array(String)` with `min_items = 1` AND `pattern = "..."`) + constraint +- [ ] `mdvs export-jsonschema` emits `minItems` / `maxItems` at the array + property level +- [ ] `mdvs init --from-jsonschema` of a schema with `minItems` / `maxItems` + produces a config that re-exports identically +- [ ] Coexistence with per-element constraints works (e.g. `Array(String)` with + `min_items = 1` AND `pattern = "..."`) - [ ] Docs updated (book + spec) ## Out of scope -- Inferring `min_items` / `max_items` from observed data (future TODO - if users ask) +- Inferring `min_items` / `max_items` from observed data (future TODO if users + ask) - `uniqueItems` JSON Schema keyword (different feature) - `prefixItems` / tuple validation (gate already hard-rejects this) -- Length constraints on nested array dimensions (e.g. - `Array(Array(Integer))` with bounds on the inner arrays) — out of - scope; today's per-element constraint shape doesn't compose into - nested arrays either +- Length constraints on nested array dimensions (e.g. `Array(Array(Integer))` + with bounds on the inner arrays) — out of scope; today's per-element + constraint shape doesn't compose into nested arrays either diff --git a/docs/spec/todos/TODO-0165.md b/docs/spec/todos/TODO-0165.md index 784853c..7ed5eb3 100644 --- a/docs/spec/todos/TODO-0165.md +++ b/docs/spec/todos/TODO-0165.md @@ -12,21 +12,21 @@ blocks: [] ## Summary -mdvs's validation engine is `jsonschema` 0.46 (JSON Schema 2020-12), but -the DSL (`mdvs.toml` `[[fields.field]]` + `[fields.field.constraints]`) -exposes only a fraction of what the spec supports. Some keywords are -**allow-listed** (the gate accepts them, the validator wires them up -end-to-end via `--from-jsonschema`) but **never emitted** by -`dsl_to_canonical` — so the runtime is half-wired and a user writing -`mdvs.toml` by hand can't reach them. Others aren't recognized at all -and would need both an allow-list addition and a DSL knob. - -This TODO inventories every JSON Schema keyword **not** currently -exposed via the DSL, records the recommendation (adopt / skip / defer), -and serves as the parent issue for the keyword-specific work. When a -keyword is adopted, split it into its own TODO and link back here. +mdvs's validation engine is `jsonschema` 0.46 (JSON Schema 2020-12), but the DSL +(`mdvs.toml` `[[fields.field]]` + `[fields.field.constraints]`) exposes only a +fraction of what the spec supports. Some keywords are **allow-listed** (the gate +accepts them, the validator wires them up end-to-end via `--from-jsonschema`) +but **never emitted** by `dsl_to_canonical` — so the runtime is half-wired and a +user writing `mdvs.toml` by hand can't reach them. Others aren't recognized at +all and would need both an allow-list addition and a DSL knob. + +This TODO inventories every JSON Schema keyword **not** currently exposed via +the DSL, records the recommendation (adopt / skip / defer), and serves as the +parent issue for the keyword-specific work. When a keyword is adopted, split it +into its own TODO and link back here. Companion TODOs already opened from this survey: + - [TODO-0164](TODO-0164.md) — `minItems` / `maxItems` (medium priority) ## Methodology @@ -45,123 +45,115 @@ For each keyword: ### Already adopted -These are emitted by `dsl_to_canonical` today; listed for completeness -only: - -| Keyword | DSL surface | -|---|---| -| `type` | `type = "..."` | -| `properties` | nested via dotted-name flattening | -| `additionalProperties` | always `true` (deliberate; nested object permissiveness) | -| `items` | per-element constraint target on `Array(*)` fields | -| `enum` | `constraints.categories` | -| `minimum` / `maximum` | `constraints.min` / `constraints.max` | -| `minLength` / `maxLength` | `constraints.min_length` / `constraints.max_length` | -| `pattern` | `constraints.pattern` | -| `format` (`date`, `date-time`) | implicit via `FieldType::Date` / `FieldType::DateTime` | +These are emitted by `dsl_to_canonical` today; listed for completeness only: + +| Keyword | DSL surface | +| ------------------------------ | -------------------------------------------------------- | +| `type` | `type = "..."` | +| `properties` | nested via dotted-name flattening | +| `additionalProperties` | always `true` (deliberate; nested object permissiveness) | +| `items` | per-element constraint target on `Array(*)` fields | +| `enum` | `constraints.categories` | +| `minimum` / `maximum` | `constraints.min` / `constraints.max` | +| `minLength` / `maxLength` | `constraints.min_length` / `constraints.max_length` | +| `pattern` | `constraints.pattern` | +| `format` (`date`, `date-time`) | implicit via `FieldType::Date` / `FieldType::DateTime` | ### Allow-listed but never emitted by the DSL -These keywords pass the gate (an imported schema may carry them and -they'll validate), but there's no `mdvs.toml` way to declare them. +These keywords pass the gate (an imported schema may carry them and they'll +validate), but there's no `mdvs.toml` way to declare them. #### `exclusiveMinimum` / `exclusiveMaximum` - **Expresses:** strict (`>`, `<`) numeric bounds vs. the inclusive `minimum`/`maximum` -- **Cost:** add `exclusive_min` / `exclusive_max` fields to - `Constraints` (or repurpose `min`/`max` with a `range_kind` - discriminator — less clean); both `Float` and `Integer` plus the - parallel `Array(Float)` / `Array(Integer)` element-wise; mutual - exclusion check (`min` and `exclusive_min` can't coexist) -- **Value:** "positive float" without the `> 0.000...01` workaround. - "Strictly less than the ceiling." Common in scientific data +- **Cost:** add `exclusive_min` / `exclusive_max` fields to `Constraints` (or + repurpose `min`/`max` with a `range_kind` discriminator — less clean); both + `Float` and `Integer` plus the parallel `Array(Float)` / `Array(Integer)` + element-wise; mutual exclusion check (`min` and `exclusive_min` can't coexist) +- **Value:** "positive float" without the `> 0.000...01` workaround. "Strictly + less than the ceiling." Common in scientific data - **Recommendation:** **adopt — open follow-up TODO** -- **Why:** trivial extension of existing range pipeline; real use case - (Float fields representing measurements that can't equal zero or - the calibration ceiling); round-trip via `canonical_to_dsl` is - one-line each direction +- **Why:** trivial extension of existing range pipeline; real use case (Float + fields representing measurements that can't equal zero or the calibration + ceiling); round-trip via `canonical_to_dsl` is one-line each direction #### `multipleOf` -- **Expresses:** value must be a divisor of N (e.g. `Integer` - multipleOf 5 → 0, 5, 10, ...) -- **Cost:** new `multiple_of` field on `Constraints`; applies to - `Integer` / `Float` and the parallel arrays; new - `ConstraintKind::MultipleOf` -- **Value:** rare. Quantized measurements; sample-rate dividers; - page-size constraints. Useful when present but niche +- **Expresses:** value must be a divisor of N (e.g. `Integer` multipleOf 5 → 0, + 5, 10, ...) +- **Cost:** new `multiple_of` field on `Constraints`; applies to `Integer` / + `Float` and the parallel arrays; new `ConstraintKind::MultipleOf` +- **Value:** rare. Quantized measurements; sample-rate dividers; page-size + constraints. Useful when present but niche - **Recommendation:** **skip** until requested -- **Why:** very small audience; the use cases are usually domain- - specific enough that a pattern or range covers them. Easy to add - later if a user asks; no design risk in deferring +- **Why:** very small audience; the use cases are usually domain- specific + enough that a pattern or range covers them. Easy to add later if a user asks; + no design risk in deferring #### `minItems` / `maxItems` - **Expresses:** bounds on array length itself (not per-element) - **Cost:** see [TODO-0164](TODO-0164.md) for full design -- **Value:** common — "tags must have at least one entry", "no more - than 5 attendees" +- **Value:** common — "tags must have at least one entry", "no more than 5 + attendees" - **Recommendation:** **already adopted** — see TODO-0164 -- **Why:** medium priority because the per-element vs array-level - split is the right place to fix the asymmetry while we're touching - array constraint emission +- **Why:** medium priority because the per-element vs array-level split is the + right place to fix the asymmetry while we're touching array constraint + emission #### `uniqueItems` - **Expresses:** array elements must be pairwise distinct -- **Cost:** add `unique_items: Option` to `Constraints`; emit on - array property (parallel to `minItems`/`maxItems` — same emission - point); applies to `Array(*)` of scalar types (semantically dubious - on `Array(Object)`); new `ConstraintKind::UniqueItems` -- **Value:** common — "tags shouldn't repeat", "attendees shouldn't - duplicate". Fields where duplicates are likely a config bug +- **Cost:** add `unique_items: Option` to `Constraints`; emit on array + property (parallel to `minItems`/`maxItems` — same emission point); applies to + `Array(*)` of scalar types (semantically dubious on `Array(Object)`); new + `ConstraintKind::UniqueItems` +- **Value:** common — "tags shouldn't repeat", "attendees shouldn't duplicate". + Fields where duplicates are likely a config bug - **Recommendation:** **adopt — fold into TODO-0164** (they share the - array-level emission point; doing them together is cheaper than - twice) -- **Why:** small surface, real demand, natural pairing with the - min/max items work + array-level emission point; doing them together is cheaper than twice) +- **Why:** small surface, real demand, natural pairing with the min/max items + work #### `const` - **Expresses:** value must equal exactly one specified constant -- **Cost:** new `Constraints.const_value: Option`; emit - as `const` keyword -- **Value:** rare in field-validation contexts; subsumed by - `categories = ["x"]` which is one element of an enum +- **Cost:** new `Constraints.const_value: Option`; emit as `const` + keyword +- **Value:** rare in field-validation contexts; subsumed by `categories = ["x"]` + which is one element of an enum - **Recommendation:** **skip** -- **Why:** `categories` with one value already does this. No new - semantic; just a syntactic alternative +- **Why:** `categories` with one value already does this. No new semantic; just + a syntactic alternative #### `required` (root-level) -- **Expresses:** JSON Schema's native required-property mechanism — an - array of property names that must be present -- **Cost:** would conflict with mdvs's path-scoped `x-mdvs.required` - (glob list per field) +- **Expresses:** JSON Schema's native required-property mechanism — an array of + property names that must be present +- **Cost:** would conflict with mdvs's path-scoped `x-mdvs.required` (glob list + per field) - **Value:** none — path-scoping is strictly more expressive - **Recommendation:** **skip permanently** -- **Why:** the path-scoping model is deliberate; switching would lose - the per-path-pattern requirement granularity. Already documented in +- **Why:** the path-scoping model is deliberate; switching would lose the + per-path-pattern requirement granularity. Already documented in `dsl_to_canonical` ("No root-level required array — requirement is path-scoped") #### `$schema` / `$id` / `title` / `description` -- **Expresses:** schema metadata; `description` is the most useful - (doc string) +- **Expresses:** schema metadata; `description` is the most useful (doc string) - **Cost:** if adopted, would need a way to write field-level docs in - `mdvs.toml` (e.g. `description = "..."` next to `type`); emission - would write to the property subschema; reverse direction would - pull description text out + `mdvs.toml` (e.g. `description = "..."` next to `type`); emission would write + to the property subschema; reverse direction would pull description text out - **Value:** preserve human-written documentation through the - `--from-jsonschema` → `export-jsonschema` round-trip. Without this, - any docstring authored in the source JSON Schema is silently lost -- **Recommendation:** **defer** — open a separate TODO if users - importing real-world JSON Schemas complain -- **Why:** purely documentation, no validation impact, but - user-visible loss in round-trip. Low effort but no current - user-driven demand + `--from-jsonschema` → `export-jsonschema` round-trip. Without this, any + docstring authored in the source JSON Schema is silently lost +- **Recommendation:** **defer** — open a separate TODO if users importing + real-world JSON Schemas complain +- **Why:** purely documentation, no validation impact, but user-visible loss in + round-trip. Low effort but no current user-driven demand ### Not in the allow-list (rejected as "unknown keyword") @@ -170,53 +162,46 @@ These need both `ALLOW_LIST` expansion **and** a DSL knob if adopted. #### Additional `format` values (`email`, `uri`, `uuid`, `hostname`, `ipv4`, `ipv6`, `regex`, `json-pointer`) - **Expresses:** built-in semantic formats for string fields -- **Cost:** expand `ALLOWED_FORMATS` (one constant); add new - `FieldType` variants OR a new `Constraints.format` field; choose - presentation -- **Value:** sugar over hand-written regex. `email` users today must - write `pattern = '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$'` - (as in `example_kb`'s `email` field). Built-in `format = "email"` - would be friendlier, self-documenting, and use jsonschema's - validated implementation -- **Recommendation:** **defer** — design question deserves its own - TODO -- **Why:** the design choice — new `FieldType::Email` etc. vs. a - generic `Constraints.format` field — has implications for - inference, storage, and the constraint-applicability matrix. - Worth thinking through separately. Inferring `format = "email"` - from observed values would be useful but adds inference complexity +- **Cost:** expand `ALLOWED_FORMATS` (one constant); add new `FieldType` + variants OR a new `Constraints.format` field; choose presentation +- **Value:** sugar over hand-written regex. `email` users today must write + `pattern = '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$'` (as in + `example_kb`'s `email` field). Built-in `format = "email"` would be + friendlier, self-documenting, and use jsonschema's validated implementation +- **Recommendation:** **defer** — design question deserves its own TODO +- **Why:** the design choice — new `FieldType::Email` etc. vs. a generic + `Constraints.format` field — has implications for inference, storage, and the + constraint-applicability matrix. Worth thinking through separately. Inferring + `format = "email"` from observed values would be useful but adds inference + complexity #### `deprecated` -- **Expresses:** field is obsolete; instances are still valid but - flagged +- **Expresses:** field is obsolete; instances are still valid but flagged - **Cost:** add `Constraints.deprecated: Option` or - `[[fields.field]].deprecated = true`; surface as a - non-`Disallowed`, non-`MissingRequired` warning category in - `check` -- **Value:** soft-migrate vaults that have an old field name being - replaced — `mdvs check` would warn but not fail -- **Recommendation:** **defer** — needs a "warning" output channel - that doesn't exist yet (`check` is binary: violations or success) -- **Why:** the violation model has no soft/warning tier today. - Adding `deprecated` is small; adding the warning channel is a - larger design decision. Bundle them, or wait until the warning - channel is needed for another reason + `[[fields.field]].deprecated = true`; surface as a non-`Disallowed`, + non-`MissingRequired` warning category in `check` +- **Value:** soft-migrate vaults that have an old field name being replaced — + `mdvs check` would warn but not fail +- **Recommendation:** **defer** — needs a "warning" output channel that doesn't + exist yet (`check` is binary: violations or success) +- **Why:** the violation model has no soft/warning tier today. Adding + `deprecated` is small; adding the warning channel is a larger design decision. + Bundle them, or wait until the warning channel is needed for another reason #### `default` - **Expresses:** a value to assume when the property is missing -- **Cost:** add `[[fields.field]].default = ...`; need to decide - whether mdvs **auto-fills** the value (mutating user files? almost - certainly no) or just **reports** it -- **Value:** dubious in mdvs's context. `required` paths already say - "must be present"; non-required fields are simply absent. There's - no validation logic that benefits from knowing the default +- **Cost:** add `[[fields.field]].default = ...`; need to decide whether mdvs + **auto-fills** the value (mutating user files? almost certainly no) or just + **reports** it +- **Value:** dubious in mdvs's context. `required` paths already say "must be + present"; non-required fields are simply absent. There's no validation logic + that benefits from knowing the default - **Recommendation:** **skip** -- **Why:** conflicts with the "frontmatter is the source of truth" - model. mdvs shouldn't write to user notes; defaults can't be - validated meaningfully without doing so. If users want a default, - they put it in their template +- **Why:** conflicts with the "frontmatter is the source of truth" model. mdvs + shouldn't write to user notes; defaults can't be validated meaningfully + without doing so. If users want a default, they put it in their template #### `examples` @@ -225,8 +210,8 @@ These need both `ALLOW_LIST` expansion **and** a DSL knob if adopted. - **Value:** pure docs; could help `mdvs info` or future `mdvs explain field ` output - **Recommendation:** **skip** -- **Why:** no validation use; documentation surface (`description` - alone would be the lighter version) +- **Why:** no validation use; documentation surface (`description` alone would + be the lighter version) #### `readOnly` / `writeOnly` @@ -237,68 +222,65 @@ These need both `ALLOW_LIST` expansion **and** a DSL knob if adopted. #### `contentEncoding` / `contentMediaType` / `contentSchema` -- **Expresses:** a string value carries embedded structured content - (base64 binary, JSON-in-string, etc.) -- **Cost:** moderate — would need a "validate the embedded content - against another schema" sub-pipeline -- **Value:** rare in frontmatter; users hitting this would inline - the content or use a separate file +- **Expresses:** a string value carries embedded structured content (base64 + binary, JSON-in-string, etc.) +- **Cost:** moderate — would need a "validate the embedded content against + another schema" sub-pipeline +- **Value:** rare in frontmatter; users hitting this would inline the content or + use a separate file - **Recommendation:** **skip** ### Hard-rejected (deliberately out of scope; documented for completeness) -These are in `HARD_REJECT` with explanatory messages; no change -recommended: +These are in `HARD_REJECT` with explanatory messages; no change recommended: -| Keyword | Why rejected | -|---|---| -| `oneOf`, `anyOf`, `allOf`, `not` | Composition doesn't compose with path-scoped fields | -| `if` / `then` / `else` | Conditional schemas; out of scope | -| `$ref` / `$defs` | Reference indirection; mdvs schemas are self-contained ([TODO-0156](TODO-0156.md) tracks the Array-of-Object workaround) | -| `dependentRequired` / `dependentSchemas` | "If A is present then B is required" — interaction model incompatible with field-per-leaf | -| `patternProperties` | Properties via regex; declare each field instead | -| `prefixItems` | Tuple validation (per-position items) — `Array(*)` is uniform-type | -| `contains` | At-least-one-matching-element semantics; subsumed by per-element constraints + `min_items` | -| `propertyNames` | Constraint on property name shape; declare names explicitly | +| Keyword | Why rejected | +| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `oneOf`, `anyOf`, `allOf`, `not` | Composition doesn't compose with path-scoped fields | +| `if` / `then` / `else` | Conditional schemas; out of scope | +| `$ref` / `$defs` | Reference indirection; mdvs schemas are self-contained ([TODO-0156](TODO-0156.md) tracks the Array-of-Object workaround) | +| `dependentRequired` / `dependentSchemas` | "If A is present then B is required" — interaction model incompatible with field-per-leaf | +| `patternProperties` | Properties via regex; declare each field instead | +| `prefixItems` | Tuple validation (per-position items) — `Array(*)` is uniform-type | +| `contains` | At-least-one-matching-element semantics; subsumed by per-element constraints + `min_items` | +| `propertyNames` | Constraint on property name shape; declare names explicitly | ## Summary recommendations Open follow-up TODOs for: -- **Strict bounds** (`exclusiveMinimum` / `exclusiveMaximum`) — small, - real use case -- **`uniqueItems`** — fold into [TODO-0164](TODO-0164.md) since they - share the array-level emission point +- **Strict bounds** (`exclusiveMinimum` / `exclusiveMaximum`) — small, real use + case +- **`uniqueItems`** — fold into [TODO-0164](TODO-0164.md) since they share the + array-level emission point Defer (open if a user asks): -- Additional `format` values (`email`, `uri`, `uuid`, …) — design - question worth its own discussion -- `description` round-trip — preserves user-authored docs in - `--from-jsonschema` / `export-jsonschema` +- Additional `format` values (`email`, `uri`, `uuid`, …) — design question worth + its own discussion +- `description` round-trip — preserves user-authored docs in `--from-jsonschema` + / `export-jsonschema` - `deprecated` — needs a warning channel in `check` first Skip: -- `multipleOf`, `const`, `default`, `examples`, `readOnly` / - `writeOnly`, `contentEncoding` / `contentMediaType` / - `contentSchema`, root-level `required` +- `multipleOf`, `const`, `default`, `examples`, `readOnly` / `writeOnly`, + `contentEncoding` / `contentMediaType` / `contentSchema`, root-level + `required` Permanently out of scope (documented above): -- Composition, conditionals, references, dependents, - `patternProperties`, `prefixItems`, `contains`, `propertyNames` +- Composition, conditionals, references, dependents, `patternProperties`, + `prefixItems`, `contains`, `propertyNames` ## Files this would touch (when items are picked up) -- `crates/mdvs/src/schema/constraints/mod.rs` — new fields per - adopted keyword -- `crates/mdvs/src/schema/constraints/*.rs` — per-keyword applicability - modules -- `crates/mdvs/src/schema/json_schema.rs` — emission + reverse + - optionally `ALLOW_LIST` / `ALLOWED_FORMATS` expansion -- `crates/mdvs/src/cmd/check.rs` — error mapping (most variants - already handled exhaustively) -- `crates/mdvs/src/schema/config.rs` — `MdvsToml::validate` - per-constraint applicability +- `crates/mdvs/src/schema/constraints/mod.rs` — new fields per adopted keyword +- `crates/mdvs/src/schema/constraints/*.rs` — per-keyword applicability modules +- `crates/mdvs/src/schema/json_schema.rs` — emission + reverse + optionally + `ALLOW_LIST` / `ALLOWED_FORMATS` expansion +- `crates/mdvs/src/cmd/check.rs` — error mapping (most variants already handled + exhaustively) +- `crates/mdvs/src/schema/config.rs` — `MdvsToml::validate` per-constraint + applicability - Book + spec docs diff --git a/docs/spec/todos/TODO-0166.md b/docs/spec/todos/TODO-0166.md index 94975ae..a065779 100644 --- a/docs/spec/todos/TODO-0166.md +++ b/docs/spec/todos/TODO-0166.md @@ -12,98 +12,85 @@ blocks: [] ## Summary -We don't have measured numbers for how mdvs performs against another -widely-used CLI tool in the same space. Without that, we can't tell -whether the design choices (Model2Vec static embeddings, LanceDB -native vector + FTS, hybrid via RRF) are translating into the latency -and footprint advantages they were meant to deliver — or whether we -have regressions we don't know about. **QMD** is the natural point of -comparison: it's the most established tool in this category, widely -adopted, with an architecture deliberately different from mdvs's -(GGUF runtime + LLM reranking vs static embeddings + RRF). - -This TODO sets up the measurement: defined corpus, defined queries, -defined metrics, reproducible scripts, and a report. The goal is to -**understand the product's behavior**, not to produce comparison -marketing. +We don't have measured numbers for how mdvs performs against another widely-used +CLI tool in the same space. Without that, we can't tell whether the design +choices (Model2Vec static embeddings, LanceDB native vector + FTS, hybrid via +RRF) are translating into the latency and footprint advantages they were meant +to deliver — or whether we have regressions we don't know about. **QMD** is the +natural point of comparison: it's the most established tool in this category, +widely adopted, with an architecture deliberately different from mdvs's (GGUF +runtime + LLM reranking vs static embeddings + RRF). + +This TODO sets up the measurement: defined corpus, defined queries, defined +metrics, reproducible scripts, and a report. The goal is to **understand the +product's behavior**, not to produce comparison marketing. ## Why now Two reasons: -1. **Engineering visibility.** Architecture decisions like Model2Vec - over GGUF and Lance over Parquet were made on the assumption of - specific latency / footprint properties. We haven't actually - verified those assumptions against a real comparison point. If - mdvs is slower than expected, we want to know *now*, before users - notice -2. **Honest self-assessment.** Building in isolation is a recipe for - blind spots. A side-by-side measurement against an established - alternative forces us to look at things we'd otherwise skip - (resident memory, token cost of output, setup time) and tells us - where mdvs's design tradeoffs land — including where they're - worse, not just where they're better +1. **Engineering visibility.** Architecture decisions like Model2Vec over GGUF + and Lance over Parquet were made on the assumption of specific latency / + footprint properties. We haven't actually verified those assumptions against + a real comparison point. If mdvs is slower than expected, we want to know + _now_, before users notice +2. **Honest self-assessment.** Building in isolation is a recipe for blind + spots. A side-by-side measurement against an established alternative forces + us to look at things we'd otherwise skip (resident memory, token cost of + output, setup time) and tells us where mdvs's design tradeoffs land — + including where they're worse, not just where they're better ## Metrics to measure -For each tool, on the same corpus, on the same machine, in a clean -shell: - -1. **Steady-state search latency** — wall-clock time for a query, - measured warm (the user has run a search at least once recently; - OS page cache holds the relevant files and models). Median of N - iterations per query. This is what users experience for repeated - queries -2. **Peak resident set size during search** — captured via - `/usr/bin/time -l` (macOS) or platform equivalent. Tells us how - much RAM each tool actually consumes per query -3. **CPU utilization during search** — derived as - `(user + sys) / wall × 100`. Distinguishes "CPU was working" from - "process was waiting on I/O" -4. **Index build time** — from zero to "ready to serve queries", end - to end (model download excluded; report separately as one-time - setup) -5. **Index size on disk** — `du -sh` of the artifact directory after - build -6. **Output token count** — count tokens (via `tiktoken` or similar) - of the result snippets emitted for a fixed N = 10 query. Relevant - wherever results get piped into a downstream LLM -7. **Tool footprint on disk** — disk usage of the binary + cached - models + any runtime deps. One-time, not per query - -Reported as a table per metric × tool. Search-latency metrics report -the median across iterations; raw measurements retained in the JSON -results file for inspection. +For each tool, on the same corpus, on the same machine, in a clean shell: + +1. **Steady-state search latency** — wall-clock time for a query, measured warm + (the user has run a search at least once recently; OS page cache holds the + relevant files and models). Median of N iterations per query. This is what + users experience for repeated queries +2. **Peak resident set size during search** — captured via `/usr/bin/time -l` + (macOS) or platform equivalent. Tells us how much RAM each tool actually + consumes per query +3. **CPU utilization during search** — derived as `(user + sys) / wall × 100`. + Distinguishes "CPU was working" from "process was waiting on I/O" +4. **Index build time** — from zero to "ready to serve queries", end to end + (model download excluded; report separately as one-time setup) +5. **Index size on disk** — `du -sh` of the artifact directory after build +6. **Output token count** — count tokens (via `tiktoken` or similar) of the + result snippets emitted for a fixed N = 10 query. Relevant wherever results + get piped into a downstream LLM +7. **Tool footprint on disk** — disk usage of the binary + cached models + any + runtime deps. One-time, not per query + +Reported as a table per metric × tool. Search-latency metrics report the median +across iterations; raw measurements retained in the JSON results file for +inspection. ### Why no cold-start metric -First-invocation latency (process started fresh, model not in OS -page cache) is excluded by design. Cold-start time is dominated by -**model size on disk** — and the architectural difference there is -~10–60× (mdvs's 30 MB Model2Vec vs QMD's 300 MB embedder, or 2 GB -with reranker + query expansion). Reporting a cold-start gap would -just restate the model-size delta. It's a true number but not an -engine-quality measurement, and it's also network-dependent for -first-ever runs. Warm latency is what users actually experience for -repeated queries and isolates engine behavior from model-loading -mechanics. +First-invocation latency (process started fresh, model not in OS page cache) is +excluded by design. Cold-start time is dominated by **model size on disk** — and +the architectural difference there is ~10–60× (mdvs's 30 MB Model2Vec vs QMD's +300 MB embedder, or 2 GB with reranker + query expansion). Reporting a +cold-start gap would just restate the model-size delta. It's a true number but +not an engine-quality measurement, and it's also network-dependent for +first-ever runs. Warm latency is what users actually experience for repeated +queries and isolates engine behavior from model-loading mechanics. ## Test corpus -- **Primary corpus** — a publicly available markdown corpus large - enough to exercise the real workload. Candidate: the **Kubernetes - documentation** repository (`kubernetes/website`'s - `content/en/docs` directory — thousands of `.md` files with - consistent frontmatter, technical prose, broad topic coverage). - Public, reproducible, no privacy concerns. Final choice to be +- **Primary corpus** — a publicly available markdown corpus large enough to + exercise the real workload. Candidate: the **Kubernetes documentation** + repository (`kubernetes/website`'s `content/en/docs` directory — thousands of + `.md` files with consistent frontmatter, technical prose, broad topic + coverage). Public, reproducible, no privacy concerns. Final choice to be confirmed before running the primary benchmark -- **Secondary corpus** — `example_kb` (45 files, mixed YAML / TOML / - JSON, 50 fields), to characterize "small vault" behavior, exercise - the multi-format scan path, and serve as the runner-script - validation case -- **Optional stress corpus** — a synthetic 10K-file vault to push - past the `VECTOR_INDEX_MIN_ROWS = 10_000` threshold so the IVF-PQ - vector index kicks in. Useful only if there's time; not blocking +- **Secondary corpus** — `example_kb` (45 files, mixed YAML / TOML / JSON, 50 + fields), to characterize "small vault" behavior, exercise the multi-format + scan path, and serve as the runner-script validation case +- **Optional stress corpus** — a synthetic 10K-file vault to push past the + `VECTOR_INDEX_MIN_ROWS = 10_000` threshold so the IVF-PQ vector index kicks + in. Useful only if there's time; not blocking ## Query set @@ -111,15 +98,14 @@ Five queries per corpus, chosen to be: - One **broad semantic query** (concept appears in many notes) - One **narrow semantic query** (concept appears in 1–3 notes) -- One **exact-phrase query** (BM25 strength — exercises fulltext - mode) -- One **metadata-filtered query** with `--where` (mdvs only — record - separately; not used for relative comparison) -- One **multi-word vague query** (typical user input — exercises - hybrid mode's RRF reranking) +- One **exact-phrase query** (BM25 strength — exercises fulltext mode) +- One **metadata-filtered query** with `--where` (mdvs only — record separately; + not used for relative comparison) +- One **multi-word vague query** (typical user input — exercises hybrid mode's + RRF reranking) -Run each query 3× per tool and report the median. Cold runs require -dropping the page cache between runs. +Run each query 3× per tool and report the median. Cold runs require dropping the +page cache between runs. ## Hardware and environment @@ -130,36 +116,32 @@ Document: - mdvs version (currently v0.6.2) - QMD version (whatever is current at benchmark time) - Node version (for QMD) -- Whether the QMD GGUF models are local on disk before the run - begins (they should be — model download is one-time setup, not - per-query, and is not what we're measuring) +- Whether the QMD GGUF models are local on disk before the run begins (they + should be — model download is one-time setup, not per-query, and is not what + we're measuring) -If we run on multiple machines (laptop + a faster desktop), report -both — that's useful signal about how the gap scales. +If we run on multiple machines (laptop + a faster desktop), report both — that's +useful signal about how the gap scales. ## Reading the numbers fairly -The two tools have meaningfully different feature sets. Any report -should note this so we don't draw conclusions the measurement can't -support: - -- QMD does **LLM reranking** on top of BM25 + vector; mdvs does - RRF. The reranking step changes both latency and result quality — - it isn't free, and it isn't reproducible from latency numbers - alone -- QMD does **AST-aware chunking** for source code; mdvs uses prose - chunking via `text-splitter`'s MarkdownSplitter. On code-heavy - corpora the chunking strategies will produce different - granularity, independent of search engine speed -- mdvs has **frontmatter validation** and **SQL filtering via - `--where`**; QMD doesn't. These aren't measured by any of the - seven metrics above — they're feature presence, not performance -- Different embedding dimensionality, different chunk sizes, - different ranking algorithms → not a like-for-like quality - comparison. The benchmark is about **latency, footprint, and - setup cost** under each tool's chosen defaults. Quality - comparison is a separate effort and would need a labeled query - set +The two tools have meaningfully different feature sets. Any report should note +this so we don't draw conclusions the measurement can't support: + +- QMD does **LLM reranking** on top of BM25 + vector; mdvs does RRF. The + reranking step changes both latency and result quality — it isn't free, and it + isn't reproducible from latency numbers alone +- QMD does **AST-aware chunking** for source code; mdvs uses prose chunking via + `text-splitter`'s MarkdownSplitter. On code-heavy corpora the chunking + strategies will produce different granularity, independent of search engine + speed +- mdvs has **frontmatter validation** and **SQL filtering via `--where`**; QMD + doesn't. These aren't measured by any of the seven metrics above — they're + feature presence, not performance +- Different embedding dimensionality, different chunk sizes, different ranking + algorithms → not a like-for-like quality comparison. The benchmark is about + **latency, footprint, and setup cost** under each tool's chosen defaults. + Quality comparison is a separate effort and would need a labeled query set ## Output artifacts @@ -170,43 +152,39 @@ When this TODO closes: - The exact commands run for each tool (copy-pasteable) - The "reading the numbers fairly" section above - Hardware + version details for reproducibility -- The raw measurement scripts (shell or PEP 723 Python) committed - alongside the report so the comparison can be re-run -- Optional: a chart (PNG, generated from the table) if the numbers - visualize well. SVG or PNG checked into - `docs/benchmarks/assets/` +- The raw measurement scripts (shell or PEP 723 Python) committed alongside the + report so the comparison can be re-run +- Optional: a chart (PNG, generated from the table) if the numbers visualize + well. SVG or PNG checked into `docs/benchmarks/assets/` ## Acceptance criteria -- [ ] Benchmark report committed under `docs/benchmarks/` with the - metrics table, commands, and "reading the numbers fairly" - section -- [ ] All seven metrics measured for both tools on at least the - primary corpus -- [ ] Search-latency numbers are warm/steady-state (cold-start - excluded by design — see "Why no cold-start metric") +- [ ] Benchmark report committed under `docs/benchmarks/` with the metrics + table, commands, and "reading the numbers fairly" section +- [ ] All seven metrics measured for both tools on at least the primary corpus +- [ ] Search-latency numbers are warm/steady-state (cold-start excluded by + design — see "Why no cold-start metric") - [ ] Hardware + version details recorded for reproducibility - [ ] Scripts committed so anyone can re-run the comparison -- [ ] If mdvs underperforms on any metric, the report says so - plainly +- [ ] If mdvs underperforms on any metric, the report says so plainly ## Out of scope -- Optimizing mdvs in response to benchmark findings — that's - follow-up work. This TODO is the **measurement**, not the tuning -- Comparing against any tool other than QMD — keep the scope tight - on one well-known alternative for now -- Continuous benchmark CI — one-time characterization; ongoing - perf-regression detection is a future concern -- Quality benchmarks (precision / recall on a labeled query set) — - separate, much harder work; not in this scope +- Optimizing mdvs in response to benchmark findings — that's follow-up work. + This TODO is the **measurement**, not the tuning +- Comparing against any tool other than QMD — keep the scope tight on one + well-known alternative for now +- Continuous benchmark CI — one-time characterization; ongoing perf-regression + detection is a future concern +- Quality benchmarks (precision / recall on a labeled query set) — separate, + much harder work; not in this scope ## Open questions -- Which N to use for `--limit` / `top-k`? Suggest **10**, matching - what most search consumers display -- Do we measure on a clean machine or a daily-driver? Daily-driver - is fine for the first pass; a clean cloud VM is the follow-up if - anyone questions the numbers -- Token counter choice — `tiktoken` (cl100k_base) is the de-facto - standard for "what an LLM would count". Use that +- Which N to use for `--limit` / `top-k`? Suggest **10**, matching what most + search consumers display +- Do we measure on a clean machine or a daily-driver? Daily-driver is fine for + the first pass; a clean cloud VM is the follow-up if anyone questions the + numbers +- Token counter choice — `tiktoken` (cl100k_base) is the de-facto standard for + "what an LLM would count". Use that diff --git a/docs/spec/todos/TODO-0167.md b/docs/spec/todos/TODO-0167.md index 90f0b74..7c2dcb1 100644 --- a/docs/spec/todos/TODO-0167.md +++ b/docs/spec/todos/TODO-0167.md @@ -16,71 +16,80 @@ blocks: [] ## Resolution -Implemented as designed. `Backend::search` and `LanceBackend::search` now take `query_embedding: Option>`; `Semantic` and `Hybrid` modes error if `None`, `Fulltext` ignores it. `cmd::search::run` wraps the model load and query-embed steps in `if mode != SearchMode::Fulltext`, returning `None` for the embedding when fulltext. +Implemented as designed. `Backend::search` and `LanceBackend::search` now take +`query_embedding: Option>`; `Semantic` and `Hybrid` modes error if +`None`, `Fulltext` ignores it. `cmd::search::run` wraps the model load and +query-embed steps in `if mode != SearchMode::Fulltext`, returning `None` for the +embedding when fulltext. -New regression test `fulltext_skips_model_load_and_embed_query` asserts that no `LoadModel` or `EmbedQuery` outcome appears in `steps` when mode is fulltext. +New regression test `fulltext_skips_model_load_and_embed_query` asserts that no +`LoadModel` or `EmbedQuery` outcome appears in `steps` when mode is fulltext. Measured impact on example_kb (warm cache): -- Peak RSS: ~136 MB → ~56 MB (the embedding model is no longer mapped into memory) -- Wall time: ~350 ms → ~310 ms (smaller than the predicted ~200 ms — the residual cost is LanceDB startup + FTS query, not model loading) -The wall-time delta is smaller than the prediction made when opening this TODO. The RSS drop is the more meaningful win. The remaining gap to QMD's BM25 (~150 ms warm) is now attributable to LanceDB startup + FTS-index reading, not wasted embedding work — separate optimization territory. +- Peak RSS: ~136 MB → ~56 MB (the embedding model is no longer mapped into + memory) +- Wall time: ~350 ms → ~310 ms (smaller than the predicted ~200 ms — the + residual cost is LanceDB startup + FTS query, not model loading) + +The wall-time delta is smaller than the prediction made when opening this TODO. +The RSS drop is the more meaningful win. The remaining gap to QMD's BM25 (~150 +ms warm) is now attributable to LanceDB startup + FTS-index reading, not wasted +embedding work — separate optimization territory. ## Original scope -`mdvs search --mode fulltext` currently loads the embedding model and -embeds the query before executing the BM25 search — even though the -backend's Fulltext branch never uses the embedding. Wasted work that -adds ~200 ms per query (measured on example_kb during TODO-0166 -benchmarking). +`mdvs search --mode fulltext` currently loads the embedding model and embeds the +query before executing the BM25 search — even though the backend's Fulltext +branch never uses the embedding. Wasted work that adds ~200 ms per query +(measured on example_kb during TODO-0166 benchmarking). `SearchMode::Fulltext` in `index/backend.rs` only calls -`full_text_search(fts())` — `query_embedding` is unused. So the model -load and `embedder.embed(query)` steps in `cmd/search.rs` are pure -overhead for that mode. +`full_text_search(fts())` — `query_embedding` is unused. So the model load and +`embedder.embed(query)` steps in `cmd/search.rs` are pure overhead for that +mode. ## Fix 1. `crates/mdvs/src/index/backend.rs`: - `Backend::search` and `LanceBackend::search`: change `query_embedding: Vec` → `Option>` - - `SearchMode::Semantic` and `SearchMode::Hybrid` branches: return - a clear error when the embedding is `None` (programmer error) - - `SearchMode::Fulltext` branch: unchanged — never reads the - embedding + - `SearchMode::Semantic` and `SearchMode::Hybrid` branches: return a clear + error when the embedding is `None` (programmer error) + - `SearchMode::Fulltext` branch: unchanged — never reads the embedding 2. `crates/mdvs/src/cmd/search.rs`: - - When `mode == SearchMode::Fulltext`, skip the "Load model" step - (~lines 200–253) and the "Embed query" step (~lines 255–263) + - When `mode == SearchMode::Fulltext`, skip the "Load model" step (~lines + 200–253) and the "Embed query" step (~lines 255–263) - Pass `None` to `backend.search` - - `SearchOutcome.model_name` continues to be populated from - `emb_config.name` (the configured model, not necessarily loaded) - so the output still reports what model the index was built with + - `SearchOutcome.model_name` continues to be populated from `emb_config.name` + (the configured model, not necessarily loaded) so the output still reports + what model the index was built with 3. Tests: - Existing tests should pass unchanged - - Add a test that fulltext search succeeds without invoking the - embedder (e.g. by inspecting `steps` — no `LoadModel` / - `EmbedQuery` outcomes when mode is fulltext) + - Add a test that fulltext search succeeds without invoking the embedder + (e.g. by inspecting `steps` — no `LoadModel` / `EmbedQuery` outcomes when + mode is fulltext) ## Out of scope -- Skipping the model-mismatch pre-check when mode is fulltext. The - mismatch is unrelated to actually loading the model; the check is - defensive and inexpensive. If users hit friction from it on - fulltext-only workflows, open a follow-up -- Skipping the `[embedding_model]` required-config check for fulltext. - Same reasoning — the config check is cheap and orthogonal to load - cost +- Skipping the model-mismatch pre-check when mode is fulltext. The mismatch is + unrelated to actually loading the model; the check is defensive and + inexpensive. If users hit friction from it on fulltext-only workflows, open a + follow-up +- Skipping the `[embedding_model]` required-config check for fulltext. Same + reasoning — the config check is cheap and orthogonal to load cost ## Expected impact -Roughly **~200 ms saved per `mdvs search --mode fulltext`** -invocation, based on example_kb measurements (mdvs fulltext 350 ms vs -QMD search 150 ms, where the ~200 ms delta corresponds to model load -+ query embed). The Model2Vec model itself is small (~30 MB static), -but mmap + initialization is non-trivial. Peak RSS during fulltext -search should also drop noticeably (no model loaded into memory). +Roughly **~200 ms saved per `mdvs search --mode fulltext`** invocation, based on +example_kb measurements (mdvs fulltext 350 ms vs QMD search 150 ms, where the +~200 ms delta corresponds to model load + +- query embed). The Model2Vec model itself is small (~30 MB static), but mmap + + initialization is non-trivial. Peak RSS during fulltext search should also + drop noticeably (no model loaded into memory). -Discovered while running the TODO-0166 benchmark; this fix unblocks -fair fulltext-mode comparisons. +Discovered while running the TODO-0166 benchmark; this fix unblocks fair +fulltext-mode comparisons. diff --git a/docs/spec/todos/TODO-0168.md b/docs/spec/todos/TODO-0168.md index ed45941..7beab64 100644 --- a/docs/spec/todos/TODO-0168.md +++ b/docs/spec/todos/TODO-0168.md @@ -12,17 +12,16 @@ blocks: [] ## Summary -`mdvs search --output json` currently emits the full `chunk_text` -(1024-char chunk, often 400–800 chars after whitespace normalization) -for each hit. For LLM-fed workflows (RAG pipelines, agent search, -Claude Code etc.) this is wasteful — measured during TODO-0166: an -mdvs `--limit 10` semantic query emits ~1,700 cl100k_base tokens vs -QMD's ~440 tokens for the same 10 hits. ~4× more tokens per query, -which translates to real cost and lost context budget downstream. - -QMD's design choice: emit a small windowed snippet (~3–4 lines around -a "best line") wrapped in a diff-hunk header that conveys the -positional context: +`mdvs search --output json` currently emits the full `chunk_text` (1024-char +chunk, often 400–800 chars after whitespace normalization) for each hit. For +LLM-fed workflows (RAG pipelines, agent search, Claude Code etc.) this is +wasteful — measured during TODO-0166: an mdvs `--limit 10` semantic query emits +~1,700 cl100k_base tokens vs QMD's ~440 tokens for the same 10 hits. ~4× more +tokens per query, which translates to real cost and lost context budget +downstream. + +QMD's design choice: emit a small windowed snippet (~3–4 lines around a "best +line") wrapped in a diff-hunk header that conveys the positional context: ``` @@ -17,4 @@ (16 before, 15 after) @@ -32,24 +31,23 @@ positional context: ``` -Mode-agnostic: QMD does this for vector, fulltext, and hybrid alike. -The `(N before, M after)` indicator lets the consumer know how much -content is *not* being shown, in case they want to fetch the rest. +Mode-agnostic: QMD does this for vector, fulltext, and hybrid alike. The +`(N before, M after)` indicator lets the consumer know how much content is _not_ +being shown, in case they want to fetch the rest. -This TODO adopts the same approach in mdvs — but as an opt-in output -mode, not a default change (existing consumers may depend on the -current shape). +This TODO adopts the same approach in mdvs — but as an opt-in output mode, not a +default change (existing consumers may depend on the current shape). ## Why a follow-up, not a default change -- The current `chunk_text` output is correct: each hit returns the - exact chunk that was ranked. That's the truthful answer to "what - did the search match against" -- Changing the default would break the asciinema demo, terminal UX, - and any external consumer relying on the full chunk -- Different consumers want different things: a human reading results - in a terminal wants the full chunk; an LLM agent wants the snippet - to conserve tokens +- The current `chunk_text` output is correct: each hit returns the exact chunk + that was ranked. That's the truthful answer to "what did the search match + against" +- Changing the default would break the asciinema demo, terminal UX, and any + external consumer relying on the full chunk +- Different consumers want different things: a human reading results in a + terminal wants the full chunk; an LLM agent wants the snippet to conserve + tokens ## Design @@ -88,33 +86,31 @@ New compact mode (opt-in): } ``` -The snippet object carries enough position metadata for a consumer -to reconstruct or fetch the surrounding context if it wants. +The snippet object carries enough position metadata for a consumer to +reconstruct or fetch the surrounding context if it wants. -For `--output pretty` / `--output markdown`, the snippet would -be rendered with a diff-hunk header matching QMD's shape (so users -familiar with that format have a consistent experience). +For `--output pretty` / `--output markdown`, the snippet would be rendered with +a diff-hunk header matching QMD's shape (so users familiar with that format have +a consistent experience). ### Picking the "best line" -Mode-agnostic: score each line in the chunk by **substring-match count -of the query terms** — `1.0` per query term whose lowercase form is -contained in the line's lowercase form — and pick the line with the -highest score. +Mode-agnostic: score each line in the chunk by **substring-match count of the +query terms** — `1.0` per query term whose lowercase form is contained in the +line's lowercase form — and pick the line with the highest score. This is the algorithm QMD uses (verified by reading -`src/store.ts::extractSnippet` upstream). It's dead simple, no BM25 -or embeddings involved, runs in O(lines × terms) over a single chunk -(~10 lines × ~5 terms = trivial). The trick is that it works for -all retrieval modes uniformly — semantic, fulltext, and hybrid — -because the line picker is independent of the ranker. +`src/store.ts::extractSnippet` upstream). It's dead simple, no BM25 or +embeddings involved, runs in O(lines × terms) over a single chunk (~10 lines × +~5 terms = trivial). The trick is that it works for all retrieval modes +uniformly — semantic, fulltext, and hybrid — because the line picker is +independent of the ranker. -**Fallback when no line matches** (typical for pure semantic -retrieval with zero literal overlap): +**Fallback when no line matches** (typical for pure semantic retrieval with zero +literal overlap): -- Default to the chunk's first line. The reranker / vector ranker - picked this chunk for a reason, so anchoring on its start is - better than nothing +- Default to the chunk's first line. The reranker / vector ranker picked this + chunk for a reason, so anchoring on its start is better than nothing ### Window size @@ -127,47 +123,42 @@ Default 4 lines. Configurable via `--snippet-lines N`. Capped at - `--snippet-lines N` — enable + set window size (implies `--snippet`) - Without these flags: current full-chunk behavior -Either flag should work for any output mode (`pretty`, `markdown`, -`json`). Default output stays unchanged. +Either flag should work for any output mode (`pretty`, `markdown`, `json`). +Default output stays unchanged. ## Acceptance criteria -- [ ] `mdvs search ... --snippet --output json` emits the - `snippet` object instead of `chunk_text` +- [ ] `mdvs search ... --snippet --output json` emits the `snippet` object + instead of `chunk_text` - [ ] `--snippet-lines N` controls the window size -- [ ] Default behavior unchanged (no `--snippet` → full `chunk_text` - as today) +- [ ] Default behavior unchanged (no `--snippet` → full `chunk_text` as today) - [ ] Text output renders the snippet with a clear positional header -- [ ] Re-running TODO-0166 benchmark with `--snippet` on the - `example_kb` corpus brings mdvs token count within ~50% of QMD's - (concrete validation that the change actually reduces context cost) -- [ ] Documented in `book/src/commands/search.md` and - `book/src/output.md` +- [ ] Re-running TODO-0166 benchmark with `--snippet` on the `example_kb` corpus + brings mdvs token count within ~50% of QMD's (concrete validation that the + change actually reduces context cost) +- [ ] Documented in `book/src/commands/search.md` and `book/src/output.md` ## Out of scope -- Changing the default. The default stays `chunk_text` — the snippet - is opt-in -- Changing the chunking strategy. Chunks remain 1024 chars; this - TODO only affects what we *emit* per hit -- LanceDB API extensions. The substring-scoring approach is self- - contained at the output layer — no upstream API dependency, no - retrieval-engine changes +- Changing the default. The default stays `chunk_text` — the snippet is opt-in +- Changing the chunking strategy. Chunks remain 1024 chars; this TODO only + affects what we _emit_ per hit +- LanceDB API extensions. The substring-scoring approach is self- contained at + the output layer — no upstream API dependency, no retrieval-engine changes ## Files to touch (sketch) - `crates/mdvs/src/cmd/search.rs` — CLI flag handling -- `crates/mdvs/src/outcome.rs` (or wherever `SearchHit` lives) — add - a `snippet: Option` variant; gate on flag -- `crates/mdvs/src/index/backend.rs` — return enough info to compute - the snippet (already has `start_line` / `end_line`) +- `crates/mdvs/src/outcome.rs` (or wherever `SearchHit` lives) — add a + `snippet: Option` variant; gate on flag +- `crates/mdvs/src/index/backend.rs` — return enough info to compute the snippet + (already has `start_line` / `end_line`) - `book/src/commands/search.md`, `book/src/output.md` — docs - New tests under `crates/mdvs/src/cmd/search.rs::tests` ## Related -- Discovered during TODO-0166 (benchmark vs QMD). The headline finding - was "mdvs emits ~4× more tokens than QMD per query" — this TODO is - the fix -- Adjacent to TODO-0167 (the fulltext perf fix) — both came out of - the same benchmark recon +- Discovered during TODO-0166 (benchmark vs QMD). The headline finding was "mdvs + emits ~4× more tokens than QMD per query" — this TODO is the fix +- Adjacent to TODO-0167 (the fulltext perf fix) — both came out of the same + benchmark recon diff --git a/docs/spec/todos/TODO-0169.md b/docs/spec/todos/TODO-0169.md index b3897d5..2c95f0e 100644 --- a/docs/spec/todos/TODO-0169.md +++ b/docs/spec/todos/TODO-0169.md @@ -15,45 +15,40 @@ blocks: [] ## Resolution -**Root cause identified and fixed.** mdvs was writing Lance **file format -v2.1** (the crate default). v2.1's miniblock encoder caps a single chunk -at **32 KiB** (u16 metadata). mdvs stores frontmatter as a nested Arrow -Struct containing every inferred field; on heterogeneous corpora most -fields are null for most rows, producing dense repetition/definition -levels. Lance's `repdef_too_sparse_for_miniblock` heuristic -under-estimates the rep/def buffer size, routes the column into miniblock -anyway, and a chunk overflows 32 KiB → the assertion at -`lance-encoding/src/encodings/logical/primitive.rs:3973` fires. - -It was **not** a single oversized value: the largest frontmatter field -across the corpus was 2,712 bytes, and `chunk_text` is bounded to ~1024 -chars by the MarkdownSplitter. The overflow is an aggregate miniblock -chunk, not one big value. +**Root cause identified and fixed.** mdvs was writing Lance **file format v2.1** +(the crate default). v2.1's miniblock encoder caps a single chunk at **32 KiB** +(u16 metadata). mdvs stores frontmatter as a nested Arrow Struct containing +every inferred field; on heterogeneous corpora most fields are null for most +rows, producing dense repetition/definition levels. Lance's +`repdef_too_sparse_for_miniblock` heuristic under-estimates the rep/def buffer +size, routes the column into miniblock anyway, and a chunk overflows 32 KiB → +the assertion at `lance-encoding/src/encodings/logical/primitive.rs:3973` fires. + +It was **not** a single oversized value: the largest frontmatter field across +the corpus was 2,712 bytes, and `chunk_text` is bounded to ~1024 chars by the +MarkdownSplitter. The overflow is an aggregate miniblock chunk, not one big +value. **Fix:** write Lance **v2.2** files via `data_storage_version` in -`database_options` on connect (`backend.rs::connect`). v2.2 is a stable -Lance file version that uses u32 miniblock metadata with a 4 GiB cap, so -the assertion can't fire. Only newly created tables are affected; the -same bundled Lance reads both v2.1 and v2.2, so existing indexes keep -working. - -**Build-time only.** The panic occurred during `mdvs build` → -`write_index`. Search on already-built indexes was never affected — the -bad index could not be written in the first place (atomic failure, no -corruption). - -**Verified:** full kubernetes/website `content/en/docs/` (1,669 files) -now builds cleanly (32 MB index); semantic / fulltext / hybrid search all -work on it; example_kb build + search unchanged; 806 tests pass, clippy + -fmt clean. - -Possible future hardening (not done, low value): a regression test -asserting the written format is v2.2. Skipped because reading the version -back needs either the `lance` crate as a direct dependency or the -soon-to-be-removed `Table::as_native()` API, and the actual overflow -needs ~1,669 files to reproduce (too heavy for a unit test). The -`connect()` code comment documents the rationale to guard against casual -removal. +`database_options` on connect (`backend.rs::connect`). v2.2 is a stable Lance +file version that uses u32 miniblock metadata with a 4 GiB cap, so the assertion +can't fire. Only newly created tables are affected; the same bundled Lance reads +both v2.1 and v2.2, so existing indexes keep working. + +**Build-time only.** The panic occurred during `mdvs build` → `write_index`. +Search on already-built indexes was never affected — the bad index could not be +written in the first place (atomic failure, no corruption). + +**Verified:** full kubernetes/website `content/en/docs/` (1,669 files) now +builds cleanly (32 MB index); semantic / fulltext / hybrid search all work on +it; example_kb build + search unchanged; 806 tests pass, clippy + fmt clean. + +Possible future hardening (not done, low value): a regression test asserting the +written format is v2.2. Skipped because reading the version back needs either +the `lance` crate as a direct dependency or the soon-to-be-removed +`Table::as_native()` API, and the actual overflow needs ~1,669 files to +reproduce (too heavy for a unit test). The `connect()` code comment documents +the rationale to guard against casual removal. ## What we observed @@ -70,56 +65,51 @@ called `Result::unwrap()` on an `Err` value: RecvError(()) Exit code 101. Discovered 2026-05-28 during TODO-0166 benchmarking. -Subsets `concepts/` (175 files) and `tasks/` (222 files) build -cleanly. So whatever triggers it is somewhere in the other ~1,200 -files (`reference/`, `setup/`, `tutorials/`, `contribute/`, etc.). +Subsets `concepts/` (175 files) and `tasks/` (222 files) build cleanly. So +whatever triggers it is somewhere in the other ~1,200 files (`reference/`, +`setup/`, `tutorials/`, `contribute/`, etc.). ## What we don't know yet -- **Whether it's a Lance bug, an mdvs bug, or expected behavior on - malformed input.** The assertion lives in `lance-encoding`, but the - data feeding it comes from mdvs. We could be sending Lance - something it explicitly doesn't support -- **What input shape triggers it.** "chunk_bytes <= max_chunk_size" - is the assertion message. We don't know what `chunk_bytes` actually - was or what value/column was being written -- **Which column.** Could be `chunk_text` (mdvs's 1024-char chunks - might exceed in unusual cases), a frontmatter string field - (`description`, `title`, etc.), or even the `embedding` column in - some encoding path we haven't considered -- **Which file(s) trigger it.** Could be one outlier, could be a - pattern across many files -- **Whether `max_chunk_size` is a hard limit or a configurable one.** - If Lance has a knob, we may need to set it; if not, this is a real - Lance limitation that needs upstream resolution +- **Whether it's a Lance bug, an mdvs bug, or expected behavior on malformed + input.** The assertion lives in `lance-encoding`, but the data feeding it + comes from mdvs. We could be sending Lance something it explicitly doesn't + support +- **What input shape triggers it.** "chunk_bytes <= max_chunk_size" is the + assertion message. We don't know what `chunk_bytes` actually was or what + value/column was being written +- **Which column.** Could be `chunk_text` (mdvs's 1024-char chunks might exceed + in unusual cases), a frontmatter string field (`description`, `title`, etc.), + or even the `embedding` column in some encoding path we haven't considered +- **Which file(s) trigger it.** Could be one outlier, could be a pattern across + many files +- **Whether `max_chunk_size` is a hard limit or a configurable one.** If Lance + has a knob, we may need to set it; if not, this is a real Lance limitation + that needs upstream resolution ## Investigation plan Cheap first: -1. **Bisect by subdir.** Build on growing subsets of - `content/en/docs/` until we find the failing one. Estimated 5-6 - build runs to narrow to a single subdir -2. **Bisect by file count within that subdir.** Halve until single - file -3. **Inspect the triggering file.** Dump its frontmatter byte sizes - per field; count its chunk byte sizes after text-splitter -4. **Read the lance-encoding source at the assertion site.** Identify - what `max_chunk_size` actually is and which encoding path is hit -5. **Look up `chunk_bytes` semantics** in lance-encoding docs / - source. Is this a per-value page chunk, or some other "chunk" - concept? +1. **Bisect by subdir.** Build on growing subsets of `content/en/docs/` until we + find the failing one. Estimated 5-6 build runs to narrow to a single subdir +2. **Bisect by file count within that subdir.** Halve until single file +3. **Inspect the triggering file.** Dump its frontmatter byte sizes per field; + count its chunk byte sizes after text-splitter +4. **Read the lance-encoding source at the assertion site.** Identify what + `max_chunk_size` actually is and which encoding path is hit +5. **Look up `chunk_bytes` semantics** in lance-encoding docs / source. Is this + a per-value page chunk, or some other "chunk" concept? Only after the bisection do we know: - Whether mdvs is sending something invalid (our bug) -- Whether Lance has a documented limit we're exceeding (use within - spec, configure differently) +- Whether Lance has a documented limit we're exceeding (use within spec, + configure differently) - Whether Lance has an undocumented limit (their bug, file upstream) -Do NOT prescribe a workaround until we know which of these we're -dealing with. Prematurely truncating values or skipping files could -mask a real issue. +Do NOT prescribe a workaround until we know which of these we're dealing with. +Prematurely truncating values or skipping files could mask a real issue. ## Reproduction @@ -133,20 +123,17 @@ Reliable, no flake. ## Immediate impact -TODO-0166 benchmark workaround: run on `tasks/` subset (222 files, -~5× larger than `example_kb`) instead of the full K8s docs. Reasonable -but not the full picture we wanted. +TODO-0166 benchmark workaround: run on `tasks/` subset (222 files, ~5× larger +than `example_kb`) instead of the full K8s docs. Reasonable but not the full +picture we wanted. -Real users will hit this on any sufficiently large corpus that -contains the trigger shape. The error message they see is a Rust -panic backtrace, not an mdvs-shaped error. UX is bad regardless of -root cause. +Real users will hit this on any sufficiently large corpus that contains the +trigger shape. The error message they see is a Rust panic backtrace, not an +mdvs-shaped error. UX is bad regardless of root cause. ## Out of scope (until we know more) - Filing upstream (premature; we may be the bug) -- Adding a workaround (premature; we don't know what's being - truncated/skipped) -- Changing Lance backend (drastic; we hit one assertion in months of - use) +- Adding a workaround (premature; we don't know what's being truncated/skipped) +- Changing Lance backend (drastic; we hit one assertion in months of use) - Documenting "don't run on large corpora" (a placeholder, not a fix) diff --git a/docs/spec/todos/TODO-0170.md b/docs/spec/todos/TODO-0170.md index 5d45c0b..c90f950 100644 --- a/docs/spec/todos/TODO-0170.md +++ b/docs/spec/todos/TODO-0170.md @@ -12,92 +12,89 @@ blocks: [] ## Status: deferred (2026-05-29) -Superseded for current corpora by **TODO-0172**. After precompiling -`GlobSet`s per field, hoisting `FieldType::try_from` out of the per-file -loop, and using `validator.is_valid()` as the fast path, `check::validate` -on the 1,669-file kubernetes/website corpus dropped from **~8 s to ~8 ms** -(see `crates/mdvs/examples/profile_pipeline.rs`). End-to-end `mdvs search` -on the same corpus went from ~21 s to ~2 s wall clock — the entire -motivating problem. - -The design below remains the right shape if validation ever again becomes -the dominant cost (likely at 100k+ files, or if a future feature adds -substantially more per-file validation work). Until then, the ~500 LOC of -cache implementation + correctness surface (invariants I1/I2, meta_hash -plumbing, prerequisite refactor, property tests) isn't earning rent. Keep -the doc as a record of the design conversation; reopen when the profile -shows validate creeping back above ~1 s at any realistic scale. +Superseded for current corpora by **TODO-0172**. After precompiling `GlobSet`s +per field, hoisting `FieldType::try_from` out of the per-file loop, and using +`validator.is_valid()` as the fast path, `check::validate` on the 1,669-file +kubernetes/website corpus dropped from **~8 s to ~8 ms** (see +`crates/mdvs/examples/profile_pipeline.rs`). End-to-end `mdvs search` on the +same corpus went from ~21 s to ~2 s wall clock — the entire motivating problem. + +The design below remains the right shape if validation ever again becomes the +dominant cost (likely at 100k+ files, or if a future feature adds substantially +more per-file validation work). Until then, the ~500 LOC of cache +implementation + correctness surface (invariants I1/I2, meta_hash plumbing, +prerequisite refactor, property tests) isn't earning rent. Keep the doc as a +record of the design conversation; reopen when the profile shows validate +creeping back above ~1 s at any realistic scale. The original design follows for reference. ## Problem -`mdvs search` with the default config runs `auto_update` + `auto_build` -before every query. On large corpora this is unusable: measured during -TODO-0166 on the full kubernetes/website docs (1,669 files), default- -config search takes **~21 seconds per query** (machine-load-inflated; -still seconds on a quiet machine). The same query with `--no-update ---no-build` is **~570 ms**. +`mdvs search` with the default config runs `auto_update` + `auto_build` before +every query. On large corpora this is unusable: measured during TODO-0166 on the +full kubernetes/website docs (1,669 files), default- config search takes **~21 +seconds per query** (machine-load-inflated; still seconds on a quiet machine). +The same query with `--no-update --no-build` is **~570 ms**. -| corpus | files | mdvs default search | mdvs engine-only search | -|---|---|---|---| -| example_kb | 45 | ~110 ms overhead | ~210 ms | -| K8s docs | 1,669 | **~21 s** | ~570 ms | +| corpus | files | mdvs default search | mdvs engine-only search | +| ---------- | ----- | ------------------- | ----------------------- | +| example_kb | 45 | ~110 ms overhead | ~210 ms | +| K8s docs | 1,669 | **~21 s** | ~570 ms | -Auto-update + auto-build must stay **on by default** — the target user -is the general / Obsidian-style user: drop a note, run a search, it just -works. The job is to make those auto-steps cheap, not to remove them. +Auto-update + auto-build must stay **on by default** — the target user is the +general / Obsidian-style user: drop a note, run a search, it just works. The job +is to make those auto-steps cheap, not to remove them. ## What dominates the cost -`crates/mdvs/examples/profile_pipeline.rs` measured the pipeline phase- -by-phase on the K8s corpus: +`crates/mdvs/examples/profile_pipeline.rs` measured the pipeline phase- by-phase +on the K8s corpus: -| phase | time | share | -|---|---|---| -| walk + read (cold) | 241 ms | 3% | -| full scan (read + parse, warm) | 73 ms | 1% | -| infer | 13 ms | 0.2% | -| **validate** | **7,977 ms** | **~96%** | +| phase | time | share | +| ------------------------------ | ------------ | -------- | +| walk + read (cold) | 241 ms | 3% | +| full scan (read + parse, warm) | 73 ms | 1% | +| infer | 13 ms | 0.2% | +| **validate** | **7,977 ms** | **~96%** | -Validation is essentially the whole bill. Read+parse is ~0.3 s cold; -inference is rounding error. So the only thing worth caching is the -result of `check::validate`. +Validation is essentially the whole bill. Read+parse is ~0.3 s cold; inference +is rounding error. So the only thing worth caching is the result of +`check::validate`. ## Design principles -1. **Validation/search separation preserved.** Validation today doesn't - depend on the Lance index, doesn't need the embedding model, and - works unchanged in CI on a fresh checkout. The cache is read and - written **inside `check::validate()` — the shared core**, so both - standalone `check` and `build_core`'s internal validation call get - the same incremental behavior. Callers do not branch. -2. **Per-field granularity.** A constraint tweak on field X re-validates - X across files containing X — not the whole corpus. The hot path - (repeated search with nothing changed) is the same as a coarse - design; per-field wins on schema edits. -3. **Scope is narrow.** Inference is not cached (13 ms; pure overhead - to persist). `scan` is unchanged (read+parse is sub-300 ms). No - filesystem stat machinery (we always read each scanned file and - compute its `content_hash`, so the "stat matches but bytes changed" - make-hole doesn't apply). +1. **Validation/search separation preserved.** Validation today doesn't depend + on the Lance index, doesn't need the embedding model, and works unchanged in + CI on a fresh checkout. The cache is read and written **inside + `check::validate()` — the shared core**, so both standalone `check` and + `build_core`'s internal validation call get the same incremental behavior. + Callers do not branch. +2. **Per-field granularity.** A constraint tweak on field X re-validates X + across files containing X — not the whole corpus. The hot path (repeated + search with nothing changed) is the same as a coarse design; per-field wins + on schema edits. +3. **Scope is narrow.** Inference is not cached (13 ms; pure overhead to + persist). `scan` is unchanged (read+parse is sub-300 ms). No filesystem stat + machinery (we always read each scanned file and compute its `content_hash`, + so the "stat matches but bytes changed" make-hole doesn't apply). ## Prerequisite refactor — single per-(field, file) check function -Today's validation logic is sliced "outer loop: violation kind; inner -loop: file" across three sites: +Today's validation logic is sliced "outer loop: violation kind; inner loop: +file" across three sites: - `check_frontmatter_errors` — document-level parse errors. -- `check_field_values` — walks each file's frontmatter, then for each - declared field present runs allowed-glob (`Disallowed`), strict-Float - precheck, preprocessor pipeline, and `jsonschema` value validation. -- `check_required_fields` — for each declared required field, walks - files and emits `MissingRequired` for required-globbed-but-absent. +- `check_field_values` — walks each file's frontmatter, then for each declared + field present runs allowed-glob (`Disallowed`), strict-Float precheck, + preprocessor pipeline, and `jsonschema` value validation. +- `check_required_fields` — for each declared required field, walks files and + emits `MissingRequired` for required-globbed-but-absent. -There is no single "is X relevant to F" function and no single "run all -checks for (X, F)" function. Per-field caching needs to make decisions -at the (field, file) granularity, so before adding cache logic the -validation engine must be inverted to a per-(field, file) shape: +There is no single "is X relevant to F" function and no single "run all checks +for (X, F)" function. Per-field caching needs to make decisions at the (field, +file) granularity, so before adding cache logic the validation engine must be +inverted to a per-(field, file) shape: ```rust fn is_field_relevant_to_file(field: &TomlField, file: &ScannedFile) -> bool { @@ -115,15 +112,14 @@ fn validate_field_for_file( ``` `validate_field_for_file` consolidates: Disallowed glob check, the -required-but-absent (`MissingRequired`) decision, the strict-Float -precheck, the preprocessor pipeline application, and the `jsonschema` -value-validation pass. The cache layer then has one clean seam: -"skip this call if the cached state says (file content, field hash) is -clean." +required-but-absent (`MissingRequired`) decision, the strict-Float precheck, the +preprocessor pipeline application, and the `jsonschema` value-validation pass. +The cache layer then has one clean seam: "skip this call if the cached state +says (file content, field hash) is clean." -This refactor is independently good code (consolidates today's -interleaved checks, removes the need to interleave globbing/presence -logic between two callers) and is the precondition for cache cleanliness. +This refactor is independently good code (consolidates today's interleaved +checks, removes the need to interleave globbing/presence logic between two +callers) and is the precondition for cache cleanliness. ## Cache shape — `.mdvs/cache.toml` @@ -172,244 +168,229 @@ validated = ["title", "weight"] ### Invariants -> **I1.** Every entry in `[[files.file]].validated` was checked against -> the hash currently recorded in `[[fields.field]]` for that field -> name. +> **I1.** Every entry in `[[files.file]].validated` was checked against the hash +> currently recorded in `[[fields.field]]` for that field name. -> **I2.** Every `[[files.file]]` entry was produced under the -> `meta_hash` currently recorded at the top of the cache. +> **I2.** Every `[[files.file]]` entry was produced under the `meta_hash` +> currently recorded at the top of the cache. -I1 is preserved by re-validating field X in every file before bumping -X's hash. I2 is preserved by treating any `meta_hash` mismatch as a -cold cache (drop everything, full revalidate, rewrite). +I1 is preserved by re-validating field X in every file before bumping X's hash. +I2 is preserved by treating any `meta_hash` mismatch as a cold cache (drop +everything, full revalidate, rewrite). ### Precise definition of "validated" -A field X is in file F's `validated` list **iff X's value in F was -checked by `check::validate` this run (or a prior run whose snapshot -survives via I1) and produced no violation.** +A field X is in file F's `validated` list **iff X's value in F was checked by +`check::validate` this run (or a prior run whose snapshot survives via I1) and +produced no violation.** Consequences (this matters for correctness): - **X is present in F and passes value validation** → X in validated. - **X is present in F and fails value validation** (`WrongType`, - `InvalidCategory`, `OutOfRange`, `Disallowed`, `NullNotAllowed`) → X - NOT in validated. Next run re-runs the check and re-reports. -- **X is required at F's path and absent from F** → `MissingRequired` - violation; X NOT in validated. Next run re-runs and re-reports. -- **X is allowed-globbed at F but absent from F** → no check runs; X - NOT in validated. Next run, same situation, same no-op (the - relevance test in the algorithm short-circuits cheaply). - -A file entry with `content_hash` matching and `validated = []` is a -legitimate cache hit (it records "we've seen this exact file content -under this meta_hash; no field was relevant to it"). It's not the same -as an absent entry. - -On read, `validated` is deduplicated into a set; duplicates would be -the symptom of a bug but must not cause logic errors. + `InvalidCategory`, `OutOfRange`, `Disallowed`, `NullNotAllowed`) → X NOT in + validated. Next run re-runs the check and re-reports. +- **X is required at F's path and absent from F** → `MissingRequired` violation; + X NOT in validated. Next run re-runs and re-reports. +- **X is allowed-globbed at F but absent from F** → no check runs; X NOT in + validated. Next run, same situation, same no-op (the relevance test in the + algorithm short-circuits cheaply). + +A file entry with `content_hash` matching and `validated = []` is a legitimate +cache hit (it records "we've seen this exact file content under this meta_hash; +no field was relevant to it"). It's not the same as an absent entry. + +On read, `validated` is deduplicated into a set; duplicates would be the symptom +of a bug but must not cause logic errors. ### Why content + per-field + meta hashes and nothing else -- No `mtime`/`size`. Profiling shows read+parse is negligible; we always - re-read each scanned file and compute its current `content_hash`. So - we don't need filesystem-stat machinery, and the classic "stat - matches but bytes changed" make-hole (`cp --preserve`, restore-in- - place) doesn't apply — the content hash catches it. -- No `mdvs_version` / `generated_at` in cache content. Only a `version` - integer for format migration; on mismatch we treat the cache as - absent and rebuild. +- No `mtime`/`size`. Profiling shows read+parse is negligible; we always re-read + each scanned file and compute its current `content_hash`. So we don't need + filesystem-stat machinery, and the classic "stat matches but bytes changed" + make-hole (`cp --preserve`, restore-in- place) doesn't apply — the content + hash catches it. +- No `mdvs_version` / `generated_at` in cache content. Only a `version` integer + for format migration; on mismatch we treat the cache as absent and rebuild. ## Runtime algorithm -The cache logic lives inside `check::validate()`. Standalone `check` -and `build_core`'s internal `check::validate` call both reach it; the -caller doesn't need to know. +The cache logic lives inside `check::validate()`. Standalone `check` and +`build_core`'s internal `check::validate` call both reach it; the caller doesn't +need to know. ### `--jsonschema` override bypasses the cache -When `check::validate` is invoked under a `--jsonschema` schema -override (today's `schema_override` path), the cache is **neither read -nor written** for that run. Per-field hashes derived from the override -would diverge from those of `mdvs.toml`, so successive runs with and -without the override would thrash each other. This mirrors how -`auto_update` is already gated on override-mode. +When `check::validate` is invoked under a `--jsonschema` schema override +(today's `schema_override` path), the cache is **neither read nor written** for +that run. Per-field hashes derived from the override would diverge from those of +`mdvs.toml`, so successive runs with and without the override would thrash each +other. This mirrors how `auto_update` is already gated on override-mode. ### Setup -1. Compute the current per-field fingerprint `current_hash[X]` for each - declared field X — `canonical_serialize` of the per-field tuple - (sub-schema + preprocess list), hashed with xxh3. In memory only. -2. Compute the current `current_meta_hash` over the declared field-name - set + `[fields].ignore` + the full `[scan]` block. In memory. -3. Load `.mdvs/cache.toml`. If absent, version mismatch, parse failure, - or `cache.meta_hash` ≠ `current_meta_hash` → cold cache. Discard any - loaded entries and full-validate everything. -4. Otherwise: for each declared field X, if `cache.fields.field[X].hash` - ≠ `current_hash[X]`, mark X as **must re-validate** for this run. +1. Compute the current per-field fingerprint `current_hash[X]` for each declared + field X — `canonical_serialize` of the per-field tuple (sub-schema + + preprocess list), hashed with xxh3. In memory only. +2. Compute the current `current_meta_hash` over the declared field-name set + + `[fields].ignore` + the full `[scan]` block. In memory. +3. Load `.mdvs/cache.toml`. If absent, version mismatch, parse failure, or + `cache.meta_hash` ≠ `current_meta_hash` → cold cache. Discard any loaded + entries and full-validate everything. +4. Otherwise: for each declared field X, if `cache.fields.field[X].hash` ≠ + `current_hash[X]`, mark X as **must re-validate** for this run. ### Per file F 1. Compute F's current `content_hash`. 2. Look up F's entry by path. -3. If absent OR `entry.content_hash` ≠ current → full-validate every - declared field **relevant to F**. Replace `entry.validated` with the - set of fields that passed (per the "Precise definition" above). +3. If absent OR `entry.content_hash` ≠ current → full-validate every declared + field **relevant to F**. Replace `entry.validated` with the set of fields + that passed (per the "Precise definition" above). 4. Else for each declared field X **relevant to F**: - - If X is in `entry.validated` AND X is not marked must-re-validate - → **skip** (cache hit). - - Else → validate X for F. On clean, ensure X is in - `entry.validated`; on violation, ensure X is NOT in it. + - If X is in `entry.validated` AND X is not marked must-re-validate → + **skip** (cache hit). + - Else → validate X for F. On clean, ensure X is in `entry.validated`; on + violation, ensure X is NOT in it. "Relevant to F" matches the union of today's `check_field_values` + -`check_required_fields` semantics: X is relevant to F iff X is allowed- -globbed at F, OR X is required-globbed at F, OR X appears in F's -frontmatter. The cache layer does not reinterpret this — it calls the -existing relevance test. +`check_required_fields` semantics: X is relevant to F iff X is allowed- globbed +at F, OR X is required-globbed at F, OR X appears in F's frontmatter. The cache +layer does not reinterpret this — it calls the existing relevance test. ### End of run -- For each declared field X re-validated this run, write - `current_hash[X]` into `cache.fields.field[X].hash`. +- For each declared field X re-validated this run, write `current_hash[X]` into + `cache.fields.field[X].hash`. - Write `current_meta_hash` into `cache.meta_hash`. -- Drop `cache.fields.field` entries for fields no longer in `mdvs.toml`; - drop stale field names from each file's `validated`. -- Drop entries in `cache.files.file` for files no longer scanned this - run. -- Atomic write: serialize to `.mdvs/cache.toml.tmp`, `fsync`, then - `rename` over `.mdvs/cache.toml`. +- Drop `cache.fields.field` entries for fields no longer in `mdvs.toml`; drop + stale field names from each file's `validated`. +- Drop entries in `cache.files.file` for files no longer scanned this run. +- Atomic write: serialize to `.mdvs/cache.toml.tmp`, `fsync`, then `rename` over + `.mdvs/cache.toml`. -Steady-state "nothing changed" cost on K8s: scan (~0.3 s) + a few μs -per file for hash lookups → tens of ms total. ~21 s collapses to roughly -the engine-only ~570 ms number. +Steady-state "nothing changed" cost on K8s: scan (~0.3 s) + a few μs per file +for hash lookups → tens of ms total. ~21 s collapses to roughly the engine-only +~570 ms number. ## Concurrency and crash safety -- **Atomic write only — no explicit locking.** Write to - `.mdvs/cache.toml.tmp`, `fsync` the file, `rename` over - `.mdvs/cache.toml`, then `fsync` the **parent directory** so the - rename itself is durable across crash. On Linux ext4 without the - parent-dir fsync the rename can be lost on power loss — the standard - idiom is "file fsync + dir fsync." -- **Rename failure path.** If the `rename` fails (e.g. `EXDEV` across - a bind mount that put `tmp` on a different filesystem), delete the - `.tmp`, warn once on stderr, continue with full validation. The - cache stays at its prior state. -- **Concurrent invocations** (rare in practice — two `check`s racing, - or `check` + `search`) end with last-writer-wins rename. Both writers - derived their output from the same `mdvs.toml` snapshot they each - read, so the survivor is internally consistent. The loser's - refreshed entries are simply not persisted; next run reproduces them. - No correctness hole because cache content is deterministic from - inputs. -- **Crash mid-validation.** Cache is written once at the end of a - successful run. If we crash earlier, the on-disk state still reflects - the prior consistent snapshot; next run re-validates from scratch. +- **Atomic write only — no explicit locking.** Write to `.mdvs/cache.toml.tmp`, + `fsync` the file, `rename` over `.mdvs/cache.toml`, then `fsync` the **parent + directory** so the rename itself is durable across crash. On Linux ext4 + without the parent-dir fsync the rename can be lost on power loss — the + standard idiom is "file fsync + dir fsync." +- **Rename failure path.** If the `rename` fails (e.g. `EXDEV` across a bind + mount that put `tmp` on a different filesystem), delete the `.tmp`, warn once + on stderr, continue with full validation. The cache stays at its prior state. +- **Concurrent invocations** (rare in practice — two `check`s racing, or + `check` + `search`) end with last-writer-wins rename. Both writers derived + their output from the same `mdvs.toml` snapshot they each read, so the + survivor is internally consistent. The loser's refreshed entries are simply + not persisted; next run reproduces them. No correctness hole because cache + content is deterministic from inputs. +- **Crash mid-validation.** Cache is written once at the end of a successful + run. If we crash earlier, the on-disk state still reflects the prior + consistent snapshot; next run re-validates from scratch. - **Parse failure on read.** Treat as cold cache (full validation). ## Opt-out -- **`[scan].cache`** — boolean in `mdvs.toml`, default `true`. When - `false`, no command reads or writes the cache (always full scan). -- **`mdvs init --no-cache`** is the only flag — it writes `cache = - false` into the generated `mdvs.toml`. No other command exposes a - cache flag; the toml is the single source of truth. -- **Graceful degradation** independent of the setting: if the cache is - enabled but the file is unwritable (read-only filesystem, - permissions), warn once on stderr and continue with full validation. - This is the load-bearing safety net; the explicit opt-out is a - secondary convenience. +- **`[scan].cache`** — boolean in `mdvs.toml`, default `true`. When `false`, no + command reads or writes the cache (always full scan). +- **`mdvs init --no-cache`** is the only flag — it writes `cache = false` into + the generated `mdvs.toml`. No other command exposes a cache flag; the toml is + the single source of truth. +- **Graceful degradation** independent of the setting: if the cache is enabled + but the file is unwritable (read-only filesystem, permissions), warn once on + stderr and continue with full validation. This is the load-bearing safety net; + the explicit opt-out is a secondary convenience. ## Verification needed before shipping -- **`dsl_to_canonical` carries `allowed`/`required` per property.** - Confirmed during Wave B (under `x-mdvs.allowed` / `x-mdvs.required`). - Per-field hashing of the canonical sub-schema therefore detects a - field's own glob widening. A unit test asserts this. -- **Per-field hash covers `preprocess` and the strict-Float - precheck.** The preprocessor pipeline (`coerce_to_string`, - `widen_int_to_float`) and the strict-Float subtype precheck are - Rust-side. Confirm whether `dsl_to_canonical` round-trips - `preprocess` via `x-mdvs.preprocess` — memory says yes, but the - test is a five-minute check and IS load-bearing for cache soundness. - If preprocess is NOT in the canonical sub-schema, the per-field hash - tuple must include it explicitly (per the cache shape comment), and +- **`dsl_to_canonical` carries `allowed`/`required` per property.** Confirmed + during Wave B (under `x-mdvs.allowed` / `x-mdvs.required`). Per-field hashing + of the canonical sub-schema therefore detects a field's own glob widening. A + unit test asserts this. +- **Per-field hash covers `preprocess` and the strict-Float precheck.** The + preprocessor pipeline (`coerce_to_string`, `widen_int_to_float`) and the + strict-Float subtype precheck are Rust-side. Confirm whether + `dsl_to_canonical` round-trips `preprocess` via `x-mdvs.preprocess` — memory + says yes, but the test is a five-minute check and IS load-bearing for cache + soundness. If preprocess is NOT in the canonical sub-schema, the per-field + hash tuple must include it explicitly (per the cache shape comment), and toggling `coerce_to_string` on a field MUST bump that field's hash. -- **`meta_hash` covers the field-name set, ignore set, and the full - `[scan]` block.** Five unit tests: (a) add a new declared field — - meta_hash bumps; (b) toggle ignore — bumps; (c) flip - `frontmatter_format` — bumps; (d) edit `glob` — bumps; (e) toggle - `include_bare_files` / `skip_gitignore` — bumps. -- **`collect_violations` determinism.** Today `collect_violations` - sorts by field name only; within a field the `files` vec inherits - iteration order from a `HashMap`. The merge-equivalence property - test below will be flaky without deterministic ordering. Fix as - prerequisite: sort `files` by path inside each `FieldViolation`; - sort outer Vec by `(field, kind, rule)`. (This is independently - worth shipping for observability/test-comparison reasons.) -- **Merge equivalence property test.** Random sequence of file edits + - schema edits + ignore-set edits + `[scan]` flips. Compare the - incremental cached run's diagnostics against a `--force` full- - validation run on the same end-state. They must be identical. -- **Specific scenario tests:** field rename across two runs; `mdvs - init --force` with an existing cache; `allowed`-glob narrow → widen - round-trip; `--jsonschema` override does not touch the cache. +- **`meta_hash` covers the field-name set, ignore set, and the full `[scan]` + block.** Five unit tests: (a) add a new declared field — meta_hash bumps; (b) + toggle ignore — bumps; (c) flip `frontmatter_format` — bumps; (d) edit `glob` + — bumps; (e) toggle `include_bare_files` / `skip_gitignore` — bumps. +- **`collect_violations` determinism.** Today `collect_violations` sorts by + field name only; within a field the `files` vec inherits iteration order from + a `HashMap`. The merge-equivalence property test below will be flaky without + deterministic ordering. Fix as prerequisite: sort `files` by path inside each + `FieldViolation`; sort outer Vec by `(field, kind, rule)`. (This is + independently worth shipping for observability/test-comparison reasons.) +- **Merge equivalence property test.** Random sequence of file edits + schema + edits + ignore-set edits + `[scan]` flips. Compare the incremental cached + run's diagnostics against a `--force` full- validation run on the same + end-state. They must be identical. +- **Specific scenario tests:** field rename across two runs; `mdvs init --force` + with an existing cache; `allowed`-glob narrow → widen round-trip; + `--jsonschema` override does not touch the cache. ## Implementation notes -- **Module location.** `crates/mdvs/src/cache.rs` (sibling to - `preprocess.rs`). Cross-cutting concern with its own file format, - serde types, atomic-write helper, and clean API - (`Cache::load(path)`, `cache.is_field_clean(file, field)`, - `cache.record_pass/fail(...)`, `cache.write(path)`). Depends on - `schema::{config, json_schema}` — no cycle. -- **Cache write timing.** Inside `validate()`, after every relevant - (file, field) decision is made for the run. Specifically: after - `check_frontmatter_errors` + per-(file, field) loop + cleanup, just - before returning `Ok(CheckResult)`. Even when violations exist, - clean (file, field) pairs are recorded (the cache is "current - knowledge of clean entries," not "build succeeded"). -- **`extract_leaf_schemas` reuse.** It already returns per-leaf JSON - values keyed by dotted name; per-field hashing can reuse it directly - rather than re-walking the canonical schema. -- **Honest LOC estimate.** ~500 LOC excluding tests: ~250 cache - module, ~150 for the prerequisite refactor (`validate_field_for_ - file`), ~80 integration in `validate()`, ~30 for the config flag and - graceful-degradation. Plus ~300 LOC of tests (property test alone - is substantial). +- **Module location.** `crates/mdvs/src/cache.rs` (sibling to `preprocess.rs`). + Cross-cutting concern with its own file format, serde types, atomic-write + helper, and clean API (`Cache::load(path)`, + `cache.is_field_clean(file, field)`, `cache.record_pass/fail(...)`, + `cache.write(path)`). Depends on `schema::{config, json_schema}` — no cycle. +- **Cache write timing.** Inside `validate()`, after every relevant (file, + field) decision is made for the run. Specifically: after + `check_frontmatter_errors` + per-(file, field) loop + cleanup, just before + returning `Ok(CheckResult)`. Even when violations exist, clean (file, field) + pairs are recorded (the cache is "current knowledge of clean entries," not + "build succeeded"). +- **`extract_leaf_schemas` reuse.** It already returns per-leaf JSON values + keyed by dotted name; per-field hashing can reuse it directly rather than + re-walking the canonical schema. +- **Honest LOC estimate.** ~500 LOC excluding tests: ~250 cache module, ~150 for + the prerequisite refactor (`validate_field_for_ file`), ~80 integration in + `validate()`, ~30 for the config flag and graceful-degradation. Plus ~300 LOC + of tests (property test alone is substantial). ## Open implementation questions - **Field-name semantics across the Wave C dotted-name flattening.** `mdvs.toml`'s `[[fields.field]]` names use dotted leaf form - (`calibration.baseline.wavelength`); the cache uses the same dotted - leaf form. A unit test confirms parity with `extract_leaf_schemas`. -- **`clean` wipes the cache** (it deletes `.mdvs/`). Acceptable; the - next call runs cold. -- **`mdvs init --force` does NOT wipe `.mdvs/`.** After it, the cache - may reference fields no longer in mdvs.toml; the end-of-run cleanup - drops them. Worth an explicit test. + (`calibration.baseline.wavelength`); the cache uses the same dotted leaf form. + A unit test confirms parity with `extract_leaf_schemas`. +- **`clean` wipes the cache** (it deletes `.mdvs/`). Acceptable; the next call + runs cold. +- **`mdvs init --force` does NOT wipe `.mdvs/`.** After it, the cache may + reference fields no longer in mdvs.toml; the end-of-run cleanup drops them. + Worth an explicit test. ## Out of scope - Caching inference (13 ms; pure overhead to persist). - Making `scan` itself cache-aware (read+parse is sub-300 ms). -- Touching the Lance index from the validation layer; preserved - separation is a goal, not a side effect. -- Turning auto-update / auto-build off by default — explicitly - rejected; the target is seamless general-user UX. +- Touching the Lance index from the validation layer; preserved separation is a + goal, not a side effect. +- Turning auto-update / auto-build off by default — explicitly rejected; the + target is seamless general-user UX. - File watchers / daemons. -- Folder-level Merkle hashing (directory mtimes miss in-place edits; - per-file content_hash is the correct level for change detection). -- Explicit file locking (`flock` etc.) for concurrent invocations — - atomic rename is sufficient for the rare-races case. +- Folder-level Merkle hashing (directory mtimes miss in-place edits; per-file + content_hash is the correct level for change detection). +- Explicit file locking (`flock` etc.) for concurrent invocations — atomic + rename is sufficient for the rare-races case. ## Impact -High priority and pre-promotion. A first-time user pointing mdvs at a -big docs tree and searching would see multi-second latencies and -reasonably conclude the tool is slow — when the engine itself is sub- -second. The cache leaves the auto-pipeline on by default, preserves the -validation/search architectural separation (the cache lives inside the -validation layer's shared core; standalone `check` and `build` benefit -identically), and adds one small gitignored TOML file. +High priority and pre-promotion. A first-time user pointing mdvs at a big docs +tree and searching would see multi-second latencies and reasonably conclude the +tool is slow — when the engine itself is sub- second. The cache leaves the +auto-pipeline on by default, preserves the validation/search architectural +separation (the cache lives inside the validation layer's shared core; +standalone `check` and `build` benefit identically), and adds one small +gitignored TOML file. diff --git a/docs/spec/todos/TODO-0171.md b/docs/spec/todos/TODO-0171.md index 10768a8..145e03b 100644 --- a/docs/spec/todos/TODO-0171.md +++ b/docs/spec/todos/TODO-0171.md @@ -17,17 +17,36 @@ related: ## Summary -Semantic search is a **discovery** tool, not a graph. This TODO defines the **workflows** that use semantic similarity — nearest-neighbour and clustering over the embeddings mdvs already stores — to *surface candidate connections*, which a user or agent then **commits as explicit links** (wikilinks / `is_reference` frontmatter values). Those explicit links are the durable graph edges from [TODO-0106](TODO-0106.md); the semantic signal itself is never stored as an edge. This is mdvs's expression of *discover semantically → commit referentially*: similarity proposes, the author disposes. +Semantic search is a **discovery** tool, not a graph. This TODO defines the +**workflows** that use semantic similarity — nearest-neighbour and clustering +over the embeddings mdvs already stores — to _surface candidate connections_, +which a user or agent then **commits as explicit links** (wikilinks / +`is_reference` frontmatter values). Those explicit links are the durable graph +edges from [TODO-0106](TODO-0106.md); the semantic signal itself is never stored +as an edge. This is mdvs's expression of _discover semantically → commit +referentially_: similarity proposes, the author disposes. ## Motivation -The explicit-link graph in TODO-0106 captures only the structure a writer remembered to encode. Real notebooks are full of related-but-unlinked material — different vocabulary for the same topic, parallel projects, shared references. Surfacing those and helping the author turn them into real links is the main thing a search-aware notes tool can offer that a hand-curated graph cannot. But the connection only becomes trustworthy and queryable once it is *written down* — a cosine score is a hypothesis, not a fact. +The explicit-link graph in TODO-0106 captures only the structure a writer +remembered to encode. Real notebooks are full of related-but-unlinked material — +different vocabulary for the same topic, parallel projects, shared references. +Surfacing those and helping the author turn them into real links is the main +thing a search-aware notes tool can offer that a hand-curated graph cannot. But +the connection only becomes trustworthy and queryable once it is _written down_ +— a cosine score is a hypothesis, not a fact. ## Principle — semantic proposes, the author commits -- **Semantic similarity is ephemeral discovery.** Computed on the fly from stored embeddings; never persisted as an edge, never counted by the graph, never a source of truth. -- **Explicit links are the truth.** A suggestion becomes a graph edge only when a human or agent writes a `[[wikilink]]` or an `is_reference` frontmatter value. TODO-0106 then picks it up. -- The stored graph therefore contains only what someone deliberately linked. Recall is bounded by author diligence — which is the point: the graph asserts only vetted connections. +- **Semantic similarity is ephemeral discovery.** Computed on the fly from + stored embeddings; never persisted as an edge, never counted by the graph, + never a source of truth. +- **Explicit links are the truth.** A suggestion becomes a graph edge only when + a human or agent writes a `[[wikilink]]` or an `is_reference` frontmatter + value. TODO-0106 then picks it up. +- The stored graph therefore contains only what someone deliberately linked. + Recall is bounded by author diligence — which is the point: the graph asserts + only vetted connections. ## Workflows @@ -35,63 +54,117 @@ Three, in rough order of value. ### 1. Find related — "what should this link to?" -`search --related-to ` (from TODO-0106) → a ranked list of the most cosine-similar other files → the author reviews and adds `[[links]]` to the ones that are genuinely related. One `nearest_to` over the existing index; model-free for an existing file (reads its stored embedding). This is the primitive the other two build on. +`search --related-to ` (from TODO-0106) → a ranked list of the most +cosine-similar other files → the author reviews and adds `[[links]]` to the ones +that are genuinely related. One `nearest_to` over the existing index; model-free +for an existing file (reads its stored embedding). This is the primitive the +other two build on. ### 2. Suggest missing links — "similar but not linked" (the high-value one) -The gap between *semantic proximity* and the *explicit graph*: pairs of files that are highly similar yet have **no** explicit link between them. Computed as `top-K nearest neighbours` **minus** `already-linked` (the explicit edges from TODO-0106). Surfaces "these two are clearly about the same thing and nobody connected them" — the suggestions most worth acting on. Needs both signals: embeddings (proximity) and the 0106 link graph (what's already linked). +The gap between _semantic proximity_ and the _explicit graph_: pairs of files +that are highly similar yet have **no** explicit link between them. Computed as +`top-K nearest neighbours` **minus** `already-linked` (the explicit edges from +TODO-0106). Surfaces "these two are clearly about the same thing and nobody +connected them" — the suggestions most worth acting on. Needs both signals: +embeddings (proximity) and the 0106 link graph (what's already linked). ### 3. Cluster the corpus — "what groups exist, and are they cross-linked?" -Group documents by embedding similarity (k-means, or community detection over the on-the-fly k-NN graph) → surface clusters of related documents → suggest either a hub / MOC note or intra-cluster cross-links for clusters that are internally under-linked. Clusters are a **render / analysis-time artifact**, computed on demand and discarded — never a stored `community` label (notes belong to multiple topics; partition labels are unstable across builds). +Group documents by embedding similarity (k-means, or community detection over +the on-the-fly k-NN graph) → surface clusters of related documents → suggest +either a hub / MOC note or intra-cluster cross-links for clusters that are +internally under-linked. Clusters are a **render / analysis-time artifact**, +computed on demand and discarded — never a stored `community` label (notes +belong to multiple topics; partition labels are unstable across builds). ## The agent loop -Because mdvs's callers are agents, these workflows are naturally an **author-assist loop**, entirely within mdvs: +Because mdvs's callers are agents, these workflows are naturally an +**author-assist loop**, entirely within mdvs: -1. mdvs **suggests** (`--related-to` / missing-links / clusters), output as JSON. -2. The agent (or human) **commits** the chosen ones by writing `[[links]]` / frontmatter refs into the files. -3. mdvs **validates + surfaces** them via `check` (broken / ambiguous) and the 0106 link graph. +1. mdvs **suggests** (`--related-to` / missing-links / clusters), output as + JSON. +2. The agent (or human) **commits** the chosen ones by writing `[[links]]` / + frontmatter refs into the files. +3. mdvs **validates + surfaces** them via `check` (broken / ambiguous) and the + 0106 link graph. -mdvs suggests, the agent commits, mdvs maintains — no LLM inside mdvs, no stored semantic edges. +mdvs suggests, the agent commits, mdvs maintains — no LLM inside mdvs, no stored +semantic edges. ## No storage -Nothing here is persisted. Nearest-neighbour and clustering are computed on demand from the embeddings already in `index.lance`. The only persisted graph is TODO-0106's explicit `links` column. This is a deliberate reversal of the earlier design (below). +Nothing here is persisted. Nearest-neighbour and clustering are computed on +demand from the embeddings already in `index.lance`. The only persisted graph is +TODO-0106's explicit `links` column. This is a deliberate reversal of the +earlier design (below). ## Command surface (open) Candidates, to settle when implementing: - `search --related-to ` — exists in TODO-0106 (nearest-neighbour). -- Missing-links — a flag on `--related-to` (e.g. `--unlinked`, restrict to non-linked neighbours) or a dedicated `mdvs suggest-links []`. -- Clustering — `mdvs cluster` / `mdvs suggest`, or folded into `info` as a corpus overview. +- Missing-links — a flag on `--related-to` (e.g. `--unlinked`, restrict to + non-linked neighbours) or a dedicated `mdvs suggest-links []`. +- Clustering — `mdvs cluster` / `mdvs suggest`, or folded into `info` as a + corpus overview. - All support `--output json` for the agent loop. ## What was rejected Earlier drafts stored the similarity signal as graph edges: -- **2026-05-30**: a `similar_files: List>` column denormalised on chunk rows. -- **2026-06-19**: top-K similarity edges as rows in a shared `edges.lance` table (`relation_type = "similarity"`), merged with explicit links in a `mdvs graph related` view. +- **2026-05-30**: a `similar_files: List>` column + denormalised on chunk rows. +- **2026-06-19**: top-K similarity edges as rows in a shared `edges.lance` table + (`relation_type = "similarity"`), merged with explicit links in a + `mdvs graph related` view. -Both are **rejected**. Storing similarity as an edge freezes a fuzzy, unstable, unverified hypothesis into structural truth — it pollutes the graph with connections nobody vetted, conflates discovery with commitment, and needs a corpus-wide recompute + staleness story for a signal that should simply be recomputed on demand. Semantic stays ephemeral; only explicit links are stored. This also drops the `edges.lance` dataset, the `[graph].similar_*` knobs, the build-time similarity pass, and `--rebuild-similarity` from those drafts. +Both are **rejected**. Storing similarity as an edge freezes a fuzzy, unstable, +unverified hypothesis into structural truth — it pollutes the graph with +connections nobody vetted, conflates discovery with commitment, and needs a +corpus-wide recompute + staleness story for a signal that should simply be +recomputed on demand. Semantic stays ephemeral; only explicit links are stored. +This also drops the `edges.lance` dataset, the `[graph].similar_*` knobs, the +build-time similarity pass, and `--rebuild-similarity` from those drafts. ## Interaction with other TODOs -- **TODO-0106 (link graph).** Hard dependency. 0171 *consumes* 0106's explicit link graph (to know what's already linked, for missing-link detection) and mdvs's embeddings (for proximity), and *feeds back* into it (committed suggestions become 0106 edges). `--related-to` is defined in 0106. -- **TODO-0170 / TODO-0157 (incremental cache / ANN optimize).** Relevant only to the *latency* of the on-the-fly nearest-neighbour queries; no storage coupling. -- **TODO-0016 (Lance swap, done).** Reuses the existing vector index; adds no dataset. +- **TODO-0106 (link graph).** Hard dependency. 0171 _consumes_ 0106's explicit + link graph (to know what's already linked, for missing-link detection) and + mdvs's embeddings (for proximity), and _feeds back_ into it (committed + suggestions become 0106 edges). `--related-to` is defined in 0106. +- **TODO-0170 / TODO-0157 (incremental cache / ANN optimize).** Relevant only to + the _latency_ of the on-the-fly nearest-neighbour queries; no storage + coupling. +- **TODO-0016 (Lance swap, done).** Reuses the existing vector index; adds no + dataset. ## Open questions -1. **Missing-link threshold.** Above what cosine score is a similar-but-unlinked pair worth suggesting? A default plus a `--min-score` override; pick on a real corpus (Refractions). -2. **Clustering algorithm** (k-means vs community detection over the k-NN graph), and whether clustering ships in v0 or after the two nearest-neighbour workflows. -3. **Agent-mode JSON shape** — what a suggestion record contains so an agent can act (source, target, score, already-linked?, suggested relation). -4. **Symmetry.** top-K isn't symmetric (A in B's top-K ≠ B in A's). For missing-links, treat a pair as a candidate if either direction is a near neighbour. +1. **Missing-link threshold.** Above what cosine score is a similar-but-unlinked + pair worth suggesting? A default plus a `--min-score` override; pick on a + real corpus (Refractions). +2. **Clustering algorithm** (k-means vs community detection over the k-NN + graph), and whether clustering ships in v0 or after the two nearest-neighbour + workflows. +3. **Agent-mode JSON shape** — what a suggestion record contains so an agent can + act (source, target, score, already-linked?, suggested relation). +4. **Symmetry.** top-K isn't symmetric (A in B's top-K ≠ B in A's). For + missing-links, treat a pair as a candidate if either direction is a near + neighbour. ## Design history -- **2026-05-30**: Opened as a similarity-edge graph — a `similar_files` column on chunk rows. -- **2026-06-19**: Rewritten to store similarity edges in TODO-0106's `edges.lance` table. -- **2026-07-03**: Reframed. Semantic similarity is **discovery only, never a stored edge** (consistent with excluding semantic from the graph in TODO-0106). This TODO is now the set of **workflows** that use nearest-neighbour + clustering to *suggest* connections a user or agent then *commits as explicit links*. Dropped the stored-edge design, `edges.lance`, the `[graph].similar_*` knobs, the build-time pass, and `--rebuild-similarity`. +- **2026-05-30**: Opened as a similarity-edge graph — a `similar_files` column + on chunk rows. +- **2026-06-19**: Rewritten to store similarity edges in TODO-0106's + `edges.lance` table. +- **2026-07-03**: Reframed. Semantic similarity is **discovery only, never a + stored edge** (consistent with excluding semantic from the graph in + TODO-0106). This TODO is now the set of **workflows** that use + nearest-neighbour + clustering to _suggest_ connections a user or agent then + _commits as explicit links_. Dropped the stored-edge design, `edges.lance`, + the `[graph].similar_*` knobs, the build-time pass, and + `--rebuild-similarity`. diff --git a/docs/spec/todos/TODO-0172.md b/docs/spec/todos/TODO-0172.md index 4218547..78c4f23 100644 --- a/docs/spec/todos/TODO-0172.md +++ b/docs/spec/todos/TODO-0172.md @@ -19,120 +19,116 @@ blocks: [] Shipped. Measured on the kubernetes/website 1,669-file corpus via `crates/mdvs/examples/profile_pipeline.rs`: -| phase | before | after | -|---|---|---| -| validate | 7,977 ms | **8 ms** | -| total (scan + infer + validate) | 8,063 ms | 94 ms | +| phase | before | after | +| ------------------------------- | -------- | -------- | +| validate | 7,977 ms | **8 ms** | +| total (scan + infer + validate) | 8,063 ms | 94 ms | -End-to-end `mdvs search "..." --mode hybrid` on the same corpus dropped -from ~21 s to ~2 s wall clock. All 885 tests pass; clippy clean. +End-to-end `mdvs search "..." --mode hybrid` on the same corpus dropped from ~21 +s to ~2 s wall clock. All 885 tests pass; clippy clean. Changes: -- `FieldMeta` struct + `build_field_metas()` precomputing per-field - `GlobSet` for `allowed`/`required` and the converted `FieldType`. Built - once at the top of `validate()` and threaded through. +- `FieldMeta` struct + `build_field_metas()` precomputing per-field `GlobSet` + for `allowed`/`required` and the converted `FieldType`. Built once at the top + of `validate()` and threaded through. - `check_field_values` + `check_required_fields` take `field_metas` and precomputed `file_paths`, replacing per-(field × file × pattern) `Glob::new(...).compile_matcher()` with `globset.is_match()`. - Fast path in `check_field_values`: `validator.is_valid(...)` first; `iter_errors().collect()` only on failure. -- `collect_violations` is now fully deterministic — files within each - violation sorted by path, outer list sorted by `(field, kind, rule)`. - `ViolationKind` got `PartialOrd, Ord` derived. +- `collect_violations` is now fully deterministic — files within each violation + sorted by path, outer list sorted by `(field, kind, rule)`. `ViolationKind` + got `PartialOrd, Ord` derived. -Side effect: **TODO-0170 demoted to `deferred`** — the cache it proposed -was sized to a problem this optimization eliminated. Re-evaluate at -100k+ files. +Side effect: **TODO-0170 demoted to `deferred`** — the cache it proposed was +sized to a problem this optimization eliminated. Re-evaluate at 100k+ files. ## Problem -The phase-by-phase profile in TODO-0166 / `examples/profile_pipeline.rs` -showed `check::validate` taking **~8 s on the 1,669-file kubernetes/ -website corpus**, ~96 % of the pipeline. Spot-reading the implementation -reveals several clearly avoidable costs in the hot loop — independent -of any caching scheme. These are first-order optimizations that should -ship before (and to soften the urgency of) the incremental cache work in -TODO-0170. +The phase-by-phase profile in TODO-0166 / `examples/profile_pipeline.rs` showed +`check::validate` taking **~8 s on the 1,669-file kubernetes/ website corpus**, +~96 % of the pipeline. Spot-reading the implementation reveals several clearly +avoidable costs in the hot loop — independent of any caching scheme. These are +first-order optimizations that should ship before (and to soften the urgency of) +the incremental cache work in TODO-0170. ## What's expensive that doesn't need to be -1. **Globs are recompiled on every call.** `check_field_values` - (`check.rs:414`) and `check_required_fields` (`check.rs:519`) call - `matches_any_glob`, which builds a fresh `Glob::new(p).ok() - .compile_matcher()` for **every** field × file × pattern combination. - On the K8s corpus that's tens of thousands of regex compilations per - `validate()`. The fix is to build a `GlobSet` per field (`allowed` - and `required`) once at the start of `validate()` and pass it down. +1. **Globs are recompiled on every call.** `check_field_values` (`check.rs:414`) + and `check_required_fields` (`check.rs:519`) call `matches_any_glob`, which + builds a fresh `Glob::new(p).ok() .compile_matcher()` for **every** field × + file × pattern combination. On the K8s corpus that's tens of thousands of + regex compilations per `validate()`. The fix is to build a `GlobSet` per + field (`allowed` and `required`) once at the start of `validate()` and pass + it down. 2. **`FieldType::try_from` runs per file.** `check.rs:431` converts the - `FieldTypeSerde` to a `FieldType` inside the per-file inner loop. - It's a pure function of the field declaration — hoist out of the loop - and compute once per field at setup. + `FieldTypeSerde` to a `FieldType` inside the per-file inner loop. It's a pure + function of the field declaration — hoist out of the loop and compute once + per field at setup. -3. **`validator.iter_errors(...).collect()` allocates even on the - common path.** The vast majority of (field, file) pairs are clean - (zero errors). `iter_errors().collect()` still walks and allocates. - `validator.is_valid(value)` short-circuits on the fast path; only - call `iter_errors` when we already know there's a failure and need - the detail for the violation message. +3. **`validator.iter_errors(...).collect()` allocates even on the common path.** + The vast majority of (field, file) pairs are clean (zero errors). + `iter_errors().collect()` still walks and allocates. + `validator.is_valid(value)` short-circuits on the fast path; only call + `iter_errors` when we already know there's a failure and need the detail for + the violation message. 4. **`dsl_to_canonical` runs twice on the build-then-validate path.** - `FieldValidators::build` calls it once (`check.rs`), and TODO-0170's - cache work will need its per-field sub-schemas too. Thread the - canonical document through once instead of recomputing. - -5. **`collect_violations` is non-deterministic in inner ordering.** - Today sorts the outer `Vec` by field name only; - the per-field `files: Vec` inherits `HashMap` - iteration order, so test output flaps across runs. Sort `files` - by path inside each violation; sort the outer vec by - `(field, kind, rule)`. This is a code-quality fix, but it's also - a prerequisite for the merge-equivalence property test TODO-0170 - needs, so it ships in this wave. + `FieldValidators::build` calls it once (`check.rs`), and TODO-0170's cache + work will need its per-field sub-schemas too. Thread the canonical document + through once instead of recomputing. + +5. **`collect_violations` is non-deterministic in inner ordering.** Today sorts + the outer `Vec` by field name only; the per-field + `files: Vec` inherits `HashMap` iteration order, so test + output flaps across runs. Sort `files` by path inside each violation; sort + the outer vec by `(field, kind, rule)`. This is a code-quality fix, but it's + also a prerequisite for the merge-equivalence property test TODO-0170 needs, + so it ships in this wave. ## Out of scope - The cache itself (TODO-0170). -- Restructuring `validate` into a per-`(field, file)` function — that's - the prerequisite refactor in TODO-0170, useful on its own but bundled - there. -- Embedding/scanning/inference performance — not on the hot path per - the profile. -- Any change to validation semantics. Diagnostics output must be - byte-identical (after the determinism fix in §5) before and after. +- Restructuring `validate` into a per-`(field, file)` function — that's the + prerequisite refactor in TODO-0170, useful on its own but bundled there. +- Embedding/scanning/inference performance — not on the hot path per the + profile. +- Any change to validation semantics. Diagnostics output must be byte-identical + (after the determinism fix in §5) before and after. ## Measurement plan -`crates/mdvs/examples/profile_pipeline.rs` already times the four -phases. Re-run on `~/Repositories/personal/kubernetes-website/content/ -en/docs` before and after each change and record: +`crates/mdvs/examples/profile_pipeline.rs` already times the four phases. Re-run +on `~/Repositories/personal/kubernetes-website/content/ en/docs` before and +after each change and record: - `validate` wall time - total pipeline wall time - any change in `infer` / `scan` (should be near zero) -Sanity-check on `example_kb` too (45 files) — the absolute cost is -small but regressions should be visible. +Sanity-check on `example_kb` too (45 files) — the absolute cost is small but +regressions should be visible. ## Verification - All existing tests pass (`cargo test`). - `cargo clippy --all-targets` clean; `cargo fmt`. -- Hand-comparison of `mdvs check example_kb` and `mdvs check - ~/Repositories/personal/kubernetes-website/content/en/docs` output - before/after — must produce identical violation lists (modulo the - determinism fix in §5, which makes flaky output stable). +- Hand-comparison of `mdvs check example_kb` and + `mdvs check ~/Repositories/personal/kubernetes-website/content/en/docs` output + before/after — must produce identical violation lists (modulo the determinism + fix in §5, which makes flaky output stable). ## Impact -The cache in TODO-0170 is a substantial design and code investment. -Knocking validate from 8 s to under 1 s through these micro-fixes -would change the urgency calculus: if the bare validate becomes fast -enough, the cache becomes a nice-to-have rather than a pre-promotion -blocker. We need to measure before committing to that work. +The cache in TODO-0170 is a substantial design and code investment. Knocking +validate from 8 s to under 1 s through these micro-fixes would change the +urgency calculus: if the bare validate becomes fast enough, the cache becomes a +nice-to-have rather than a pre-promotion blocker. We need to measure before +committing to that work. -Even if the cache still ships, these wins compound — the cache's -"steady-state nothing-changed" path still has to construct and consult -hashes, and the validators it re-invokes after schema edits benefit -from the precompiled globs and fast-path validator calls. +Even if the cache still ships, these wins compound — the cache's "steady-state +nothing-changed" path still has to construct and consult hashes, and the +validators it re-invokes after schema edits benefit from the precompiled globs +and fast-path validator calls. diff --git a/docs/spec/todos/TODO-0173.md b/docs/spec/todos/TODO-0173.md index 761ae4f..1949b8d 100644 --- a/docs/spec/todos/TODO-0173.md +++ b/docs/spec/todos/TODO-0173.md @@ -21,47 +21,47 @@ blocks: [] Shipped in two stages on `feat/validate-cheap-wins`. -**Phase A — zero-change short-circuit.** `build_core` now skips -`write_index` entirely when classify reports no chunks to add and no -files to remove. Detection uses `embedded_details` (not -`needs_embedding`) so empty-body files like Hugo `_index.md`, which -classify always marks as new, correctly trigger the skip. - -**Phase B — incremental upsert/delete/optimize.** Non-full-rebuild -writes now use a new `write_index_incremental` path on `LanceBackend`: +**Phase A — zero-change short-circuit.** `build_core` now skips `write_index` +entirely when classify reports no chunks to add and no files to remove. +Detection uses `embedded_details` (not `needs_embedding`) so empty-body files +like Hugo `_index.md`, which classify always marks as new, correctly trigger the +skip. + +**Phase B — incremental upsert/delete/optimize.** Non-full-rebuild writes now +use a new `write_index_incremental` path on `LanceBackend`: `table.delete("file_id IN (...)")` for changed + removed file_ids, -`table.add(batch)` for newly embedded chunks, `NativeTable:: -replace_schema_metadata` for the `mdvs.*` keys, and -`table.optimize(OptimizeAction::All)` so FTS + IVF-PQ pick up the new -rows. `ClassifyData` gained `removed_file_ids` to carry the delete -predicate's inputs. +`table.add(batch)` for newly embedded chunks, +`NativeTable:: replace_schema_metadata` for the `mdvs.*` keys, and +`table.optimize(OptimizeAction::All)` so FTS + IVF-PQ pick up the new rows. +`ClassifyData` gained `removed_file_ids` to carry the delete predicate's inputs. Measured on the K8s 1,669-file / 22,729-chunk corpus: -| change | write_index | vs. full overwrite | -|---|---|---| -| zero change | skipped | — | -| add 1 file | 20 ms | ~67× faster | -| remove 1 file | 8 ms | ~168× faster | +| change | write_index | vs. full overwrite | +| ------------- | ----------- | ------------------ | +| zero change | skipped | — | +| add 1 file | 20 ms | ~67× faster | +| remove 1 file | 8 ms | ~168× faster | -End-to-end `mdvs search` after a single-file edit stays sub-second -instead of cliffing back to the prior ~2 s overhead. +End-to-end `mdvs search` after a single-file edit stays sub-second instead of +cliffing back to the prior ~2 s overhead. Tests added: + - `second_build_skips_write_index_when_nothing_changed` — Phase A. - `third_build_persists_new_file_via_incremental_path` — Phase B. The probe script `examples/probe_lance_incremental.rs` also gained a -batch-delete sweep confirming that one `IN (...)` delete costs the same -~2 ms regardless of how many file_ids are in the list — the per-call -overhead, not per-file. +batch-delete sweep confirming that one `IN (...)` delete costs the same ~2 ms +regardless of how many file_ids are in the list — the per-call overhead, not +per-file. The original design follows for reference. ## Problem -`crates/mdvs/examples/profile_pipeline.rs` shows the K8s 1,669-file -corpus, on a zero-change rebuild: +`crates/mdvs/examples/profile_pipeline.rs` shows the K8s 1,669-file corpus, on a +zero-change rebuild: ``` validate : 7 ms @@ -74,14 +74,12 @@ build_core total wall : 1517 ms `backend.rs::write_index` calls `conn.create_table(...).mode(CreateTableMode::Overwrite)` and then -`build_indexes(...)` — meaning **the entire Lance dataset is rebuilt and -both indexes (FTS, IVF-PQ) recreated from scratch on every `mdvs build` -invocation**, even when zero chunks changed. The 1.3 s gap between -`mdvs default` search (~1.8 s) and `mdvs engine-only` (~0.33 s) on K8s -is essentially this single call. +`build_indexes(...)` — meaning **the entire Lance dataset is rebuilt and both +indexes (FTS, IVF-PQ) recreated from scratch on every `mdvs build` invocation**, +even when zero chunks changed. The 1.3 s gap between `mdvs default` search (~1.8 +s) and `mdvs engine-only` (~0.33 s) on K8s is essentially this single call. -LanceDB 0.29 provides the right primitives — we're just using the -wrong API. +LanceDB 0.29 provides the right primitives — we're just using the wrong API. ## Goal @@ -89,13 +87,13 @@ Two phases, both shipped together so the win is permanent: ### Phase A — zero-change short-circuit -If classify reports `needs_embedding.is_empty()` AND `removed_count == 0` -AND the existing dataset's stored `schema_hash` matches the current one, -**skip the `write_index` call entirely**. The existing dataset is -already correct. ~10 LOC plus a test. +If classify reports `needs_embedding.is_empty()` AND `removed_count == 0` AND +the existing dataset's stored `schema_hash` matches the current one, **skip the +`write_index` call entirely**. The existing dataset is already correct. ~10 LOC +plus a test. -Wins the dominant `mdvs search`-with-nothing-changed case but is a -cliff: the moment one note is added, latency jumps back up. +Wins the dominant `mdvs search`-with-nothing-changed case but is a cliff: the +moment one note is added, latency jumps back up. ### Phase B — proper incremental upsert/delete @@ -109,173 +107,165 @@ existing table -> ``` LanceDB APIs to use: + - `table.delete(predicate)` — SQL `WHERE`-style row deletion. - `table.merge_insert(&["chunk_id"])` — upsert by key. - `table.add(batch)` — append. - `table.optimize(action)` — incremental index/data compaction. -Wins the realistic "small delta" case (one note edited, search again), -which is what the Obsidian-style target user does. ~150–250 LOC. +Wins the realistic "small delta" case (one note edited, search again), which is +what the Obsidian-style target user does. ~150–250 LOC. ## Caveats — probe results (2026-05-29) -`crates/mdvs/examples/probe_lance_incremental.rs` ran against a copy of -the K8s 1,669-file / 22,729-chunk index. All four caveats are settled -favorably — the incremental rewrite is feasible with no behavior -surprises. +`crates/mdvs/examples/probe_lance_incremental.rs` ran against a copy of the K8s +1,669-file / 22,729-chunk index. All four caveats are settled favorably — the +incremental rewrite is feasible with no behavior surprises. ### 1. Schema metadata mutation — ✅ supported, 1 ms -`Table::as_native().unwrap().replace_schema_metadata(map)` accepts a -fresh `HashMap` and persists it on the existing table. -The probe bumped `mdvs.built_at` and re-read the schema; the new value -was visible immediately. Cost: 1 ms. So `BuildMetadata` updates after -incremental writes are essentially free. +`Table::as_native().unwrap().replace_schema_metadata(map)` accepts a fresh +`HashMap` and persists it on the existing table. The probe +bumped `mdvs.built_at` and re-read the schema; the new value was visible +immediately. Cost: 1 ms. So `BuildMetadata` updates after incremental writes are +essentially free. Note: the API is on `lancedb::table::NativeTable`, not the public -`lancedb::Table` wrapper — call via `as_native()`. Local backend -returns `Some(_)`; if mdvs ever supports a remote backend, that's -where the conditional belongs. +`lancedb::Table` wrapper — call via `as_native()`. Local backend returns +`Some(_)`; if mdvs ever supports a remote backend, that's where the conditional +belongs. ### 2. Row shape — ✅ one row per chunk -The Lance schema has 11 fields combining chunk-level columns -(`chunk_id`, `chunk_index`, `start_line`, `end_line`, `chunk_text`, -`embedding`) and file-level columns (`file_id`, `filepath`, -`content_hash`, `data`, `built_at`). Row count equals the chunk count -(22,729). So file-level columns are **repeated across every chunk of -the same file**, and there are no separate "file-only" rows for -empty-body files. (Confirm during implementation what happens for -files that produce zero chunks — they may simply have no rows in the -table. If so, "file removed from corpus" is detected by row-set -diff per file_id, which works fine for our purposes.) +The Lance schema has 11 fields combining chunk-level columns (`chunk_id`, +`chunk_index`, `start_line`, `end_line`, `chunk_text`, `embedding`) and +file-level columns (`file_id`, `filepath`, `content_hash`, `data`, `built_at`). +Row count equals the chunk count (22,729). So file-level columns are **repeated +across every chunk of the same file**, and there are no separate "file-only" +rows for empty-body files. (Confirm during implementation what happens for files +that produce zero chunks — they may simply have no rows in the table. If so, +"file removed from corpus" is detected by row-set diff per file_id, which works +fine for our purposes.) Keying decisions that follow: + - `merge_insert(&["chunk_id"])` for inserting/updating chunk rows. -- `delete("file_id IN (...)")` for removing or replacing all chunks of - a file. +- `delete("file_id IN (...)")` for removing or replacing all chunks of a file. ### 3. Delete API — ✅ fast - No-op delete (predicate matches no rows): 11 ms. - Single-row real delete by `file_id`: 2 ms. -Both via the public `Table::delete(predicate)` method. Lance correctly -reports post-delete row counts. The 10x ratio between no-op and real -is curious; both are negligible at this scale. Batch deletes for many -files should be a single SQL `IN (...)` call rather than a loop. +Both via the public `Table::delete(predicate)` method. Lance correctly reports +post-delete row counts. The 10x ratio between no-op and real is curious; both +are negligible at this scale. Batch deletes for many files should be a single +SQL `IN (...)` call rather than a loop. ### 4. Optimize cost — ✅ cheap, ~39 ms -`Table::optimize(OptimizeAction::All)` ran in **39 ms** on the -22k-chunk table. Compaction/prune metrics were trivial (single -1-row delete didn't fragment much). At this scale we can safely -call `optimize(All)` after every incremental write; no need for a -threshold-based scheduler. +`Table::optimize(OptimizeAction::All)` ran in **39 ms** on the 22k-chunk table. +Compaction/prune metrics were trivial (single 1-row delete didn't fragment +much). At this scale we can safely call `optimize(All)` after every incremental +write; no need for a threshold-based scheduler. -(Worth re-measuring after a larger delta — say 100 chunks -inserted+deleted — to confirm optimize stays cheap; the probe's -delta was small.) +(Worth re-measuring after a larger delta — say 100 chunks inserted+deleted — to +confirm optimize stays cheap; the probe's delta was small.) ### Bonus observation -Lance is additive — a single-row delete on the 809 MB K8s index -grew the on-disk size to 812 MB. Old fragments are kept for MVCC. -`OptimizeAction::Prune { older_than: ... }` removes them; default -retention is 7 days. Worth surfacing in a long-running `mdvs build` -loop later but not relevant for the search hot path. +Lance is additive — a single-row delete on the 809 MB K8s index grew the on-disk +size to 812 MB. Old fragments are kept for MVCC. +`OptimizeAction::Prune { older_than: ... }` removes them; default retention is 7 +days. Worth surfacing in a long-running `mdvs build` loop later but not relevant +for the search hot path. ## Caveats (original list, kept for context) -The probe checked each of these against the actual lancedb 0.29 -behavior on a copy of a real index, with notes back into this TODO. -The implementation of the rewrite is now unblocked. +The probe checked each of these against the actual lancedb 0.29 behavior on a +copy of a real index, with notes back into this TODO. The implementation of the +rewrite is now unblocked. 1. **Schema metadata mutation on an existing table.** We currently bake `BuildMetadata` into the table's Arrow schema via `with_metadata` at `create_table` time. Lance has `delete_config_keys` but the path for - *adding/updating* metadata on a live table is unclear. If we can't - update it without overwrite, we either (a) accept stale `built_at` / - `schema_hash` in metadata between full rebuilds, or (b) factor - metadata out of the Arrow schema into a sidecar file. Probe needs to - answer: which APIs exist, and what's the minimum-cost way to keep - `BuildMetadata` fresh. - -2. **Row shape: file rows vs. chunk rows.** Current storage is a single - Lance table where each row is a chunk and the file-level columns - (`file_id`, `filepath`, `data`, `content_hash`, `built_at`) are - repeated across chunks of the same file. Confirm this by reading - `build_index_batch`. If true, `merge_insert` keys cleanly on - `chunk_id` and a file replace becomes "delete by `file_id`, then add - that file's new chunks." If files have their own rows (no chunks - produced — e.g. empty bodies), confirm how those are represented and - whether `merge_insert` covers them. + _adding/updating_ metadata on a live table is unclear. If we can't update it + without overwrite, we either (a) accept stale `built_at` / `schema_hash` in + metadata between full rebuilds, or (b) factor metadata out of the Arrow + schema into a sidecar file. Probe needs to answer: which APIs exist, and + what's the minimum-cost way to keep `BuildMetadata` fresh. + +2. **Row shape: file rows vs. chunk rows.** Current storage is a single Lance + table where each row is a chunk and the file-level columns (`file_id`, + `filepath`, `data`, `content_hash`, `built_at`) are repeated across chunks of + the same file. Confirm this by reading `build_index_batch`. If true, + `merge_insert` keys cleanly on `chunk_id` and a file replace becomes "delete + by `file_id`, then add that file's new chunks." If files have their own rows + (no chunks produced — e.g. empty bodies), confirm how those are represented + and whether `merge_insert` covers them. 3. **Index staleness after partial writes.** After `delete` + `add` / - `merge_insert`, both the FTS and the IVF-PQ index reflect only the - pre-write state for the persisted index files; Lance handles fresh - rows via scan-fallback. The probe should measure: query latency on a - small delta (a few rows) before and after `optimize()`. Decide - whether to optimize after every write or only above some threshold. - -4. **`optimize` cost.** Cheap "delta compaction" vs full reindex. If - `optimize` is itself slow on a 22k-chunk table, we lose part of our - win. Probe: time `optimize(OptimizeAction::All)` and any cheaper - variant on the K8s index after a small delta. - -5. **Pre-existing latent bug — flag as a separate TODO.** `content_hash` - in `build_core` is computed on `file.content`, which is the parsed - **body** of the file, not the full file (frontmatter + body). So a - frontmatter-only edit (e.g. retyping a `tags` field) doesn't change - `content_hash` → classify reports "unchanged" → no re-embed → and now - *with* Phase A/B, no rewrite either. The frontmatter column in the - Lance row stays stale. This is independent of TODO-0173 (today's - nuclear overwrite happens to mask it because frontmatter is written - from the *current* scan even for unchanged files). After this TODO, - stale frontmatter persists in the index. **Open a follow-up TODO** - to either (a) hash body+serialized-frontmatter, or (b) detect - frontmatter changes explicitly during classify and re-update file - rows even when chunks don't need re-embedding. + `merge_insert`, both the FTS and the IVF-PQ index reflect only the pre-write + state for the persisted index files; Lance handles fresh rows via + scan-fallback. The probe should measure: query latency on a small delta (a + few rows) before and after `optimize()`. Decide whether to optimize after + every write or only above some threshold. + +4. **`optimize` cost.** Cheap "delta compaction" vs full reindex. If `optimize` + is itself slow on a 22k-chunk table, we lose part of our win. Probe: time + `optimize(OptimizeAction::All)` and any cheaper variant on the K8s index + after a small delta. + +5. **Pre-existing latent bug — flag as a separate TODO.** `content_hash` in + `build_core` is computed on `file.content`, which is the parsed **body** of + the file, not the full file (frontmatter + body). So a frontmatter-only edit + (e.g. retyping a `tags` field) doesn't change `content_hash` → classify + reports "unchanged" → no re-embed → and now _with_ Phase A/B, no rewrite + either. The frontmatter column in the Lance row stays stale. This is + independent of TODO-0173 (today's nuclear overwrite happens to mask it + because frontmatter is written from the _current_ scan even for unchanged + files). After this TODO, stale frontmatter persists in the index. **Open a + follow-up TODO** to either (a) hash body+serialized-frontmatter, or (b) + detect frontmatter changes explicitly during classify and re-update file rows + even when chunks don't need re-embedding. ## Implementation order 1. Probe script + answers to caveats 1–4 written back here. (~half a day) 2. Phase A skip + test. (~hour, lands as a small commit) 3. Phase B incremental rewrite + tests. (~1–2 days) -4. Re-bench K8s with `profile_pipeline.rs`; update `docs/benchmarks/` - results + report. +4. Re-bench K8s with `profile_pipeline.rs`; update `docs/benchmarks/` results + + report. -Phase A is staged first so we have a measurable midpoint and can roll -back to it if Phase B uncovers a Lance behavior we can't work around in -this round. +Phase A is staged first so we have a measurable midpoint and can roll back to it +if Phase B uncovers a Lance behavior we can't work around in this round. ## Verification -- `cargo test` — all existing tests pass. Build incrementality is - already covered for the `content_hash`-keyed embed-skip path; we need - new tests for: incremental delete on removed files, incremental - upsert on edited files, zero-change skip path. -- Re-run `crates/mdvs/examples/profile_pipeline.rs` on example_kb and - K8s. Target: `write_index` from 1,346 ms → tens of ms on zero-change; - proportional reduction on small-delta runs. +- `cargo test` — all existing tests pass. Build incrementality is already + covered for the `content_hash`-keyed embed-skip path; we need new tests for: + incremental delete on removed files, incremental upsert on edited files, + zero-change skip path. +- Re-run `crates/mdvs/examples/profile_pipeline.rs` on example_kb and K8s. + Target: `write_index` from 1,346 ms → tens of ms on zero-change; proportional + reduction on small-delta runs. - Search hit lists must be identical before/after for the same corpus - + query (no semantic regressions from incremental index updates). -- `docs/benchmarks/run.py` re-runs cleanly with comparable / improved - setup numbers. + - query (no semantic regressions from incremental index updates). +- `docs/benchmarks/run.py` re-runs cleanly with comparable / improved setup + numbers. ## Out of scope -- The content-hash-over-body-only bug (separate follow-up TODO; will be - opened alongside this one). +- The content-hash-over-body-only bug (separate follow-up TODO; will be opened + alongside this one). - Changing the row schema. We keep one-row-per-chunk as today. - Index sharding / multi-segment Lance configurations. -- Cross-process locking on the cache (not relevant — Lance handles - concurrent writers via its own transaction model). +- Cross-process locking on the cache (not relevant — Lance handles concurrent + writers via its own transaction model). ## Impact -Closes the last gap with QMD on default search: K8s `mdvs default` -~1.8 s → ~0.5 s after Phase A, and stays there across small deltas -after Phase B. Combined with TODO-0172, mdvs would beat QMD on every -benchmark axis (setup, engine-only search, default search, index size, -model size) on this corpus. +Closes the last gap with QMD on default search: K8s `mdvs default` ~1.8 s → ~0.5 +s after Phase A, and stays there across small deltas after Phase B. Combined +with TODO-0172, mdvs would beat QMD on every benchmark axis (setup, engine-only +search, default search, index size, model size) on this corpus. diff --git a/docs/spec/todos/TODO-0174.md b/docs/spec/todos/TODO-0174.md index 7c7a504..e6744a6 100644 --- a/docs/spec/todos/TODO-0174.md +++ b/docs/spec/todos/TODO-0174.md @@ -12,62 +12,58 @@ blocks: [] ## Problem -In `cmd/build.rs`, the per-file `content_hash` used by `classify_files` -and stored in Lance is `content_hash(&file.content)` — and `file.content` -is the parsed **body** of the markdown (post-frontmatter extraction), -not the full file. So: +In `cmd/build.rs`, the per-file `content_hash` used by `classify_files` and +stored in Lance is `content_hash(&file.content)` — and `file.content` is the +parsed **body** of the markdown (post-frontmatter extraction), not the full +file. So: -- Edit only a YAML frontmatter field (no body change) → body stays - identical → `content_hash` is unchanged → classify reports the file - as "unchanged" → no re-embed, and (after [TODO-0173](TODO-0173.md)) - no row rewrite either. The stored `data` Struct column still reflects - the *previous* frontmatter. -- Today the nuclear `CreateTableMode::Overwrite` masks this because the - row is rewritten from the fresh scan every build. TODO-0173 stops - that, which is when this bug becomes visible. +- Edit only a YAML frontmatter field (no body change) → body stays identical → + `content_hash` is unchanged → classify reports the file as "unchanged" → no + re-embed, and (after [TODO-0173](TODO-0173.md)) no row rewrite either. The + stored `data` Struct column still reflects the _previous_ frontmatter. +- Today the nuclear `CreateTableMode::Overwrite` masks this because the row is + rewritten from the fresh scan every build. TODO-0173 stops that, which is when + this bug becomes visible. ## Why it matters -Search results with `--where` SQL filters over frontmatter columns -would silently return stale matches against the previous frontmatter -state until the file's body is also edited. For a user iterating on -frontmatter (the common Obsidian flow — fixing a category, adding a -tag), this is wrong. +Search results with `--where` SQL filters over frontmatter columns would +silently return stale matches against the previous frontmatter state until the +file's body is also edited. For a user iterating on frontmatter (the common +Obsidian flow — fixing a category, adding a tag), this is wrong. ## Two candidate fixes -1. **Extend `content_hash` to cover frontmatter + body.** Hash the - *raw file bytes* (or `serialized_frontmatter + "\n" + body`) instead - of `parsed.body`. Cleanest invariant; one hash captures the whole - user-visible state of the file. +1. **Extend `content_hash` to cover frontmatter + body.** Hash the _raw file + bytes_ (or `serialized_frontmatter + "\n" + body`) instead of `parsed.body`. + Cleanest invariant; one hash captures the whole user-visible state of the + file. 2. **Track a separate `frontmatter_hash` per file.** Classify reports - "frontmatter changed but body same" as a third category that - triggers a file-row update (no re-embed, just rewriting the `data` - column). More granular — saves embedding work in the corner case - where someone only edits frontmatter — but adds bookkeeping. + "frontmatter changed but body same" as a third category that triggers a + file-row update (no re-embed, just rewriting the `data` column). More + granular — saves embedding work in the corner case where someone only edits + frontmatter — but adds bookkeeping. -(1) is simpler and aligns with the rest of the design (one content -identity per file). (2) buys a small efficiency in one narrow scenario. -I'd default to (1) unless profiling shows the embedding-skip win in -scenario (2) matters. +(1) is simpler and aligns with the rest of the design (one content identity per +file). (2) buys a small efficiency in one narrow scenario. I'd default to (1) +unless profiling shows the embedding-skip win in scenario (2) matters. ## Verification -- Unit test: scan a file, capture content_hash. Edit only the - frontmatter (e.g. flip a `draft` boolean), re-scan, capture again. - After fix: hashes differ. -- Integration test: `mdvs build` (post-TODO-0173) on a vault, edit one - file's frontmatter, `mdvs build` again, assert the file row's `data` - column matches the new frontmatter (not the old). +- Unit test: scan a file, capture content_hash. Edit only the frontmatter (e.g. + flip a `draft` boolean), re-scan, capture again. After fix: hashes differ. +- Integration test: `mdvs build` (post-TODO-0173) on a vault, edit one file's + frontmatter, `mdvs build` again, assert the file row's `data` column matches + the new frontmatter (not the old). ## Out of scope - Changing the visible `--where` semantics or the `data` Struct shape. -- Backfilling the hash for existing built indexes (any `mdvs build` - --force handles this; we don't need a migration). +- Backfilling the hash for existing built indexes (any `mdvs build` --force + handles this; we don't need a migration). ## Impact -Latent today behind the nuclear overwrite; becomes a real correctness -bug the moment TODO-0173 ships. Track as a follow-up rather than -folding into 0173 so each TODO has one job. +Latent today behind the nuclear overwrite; becomes a real correctness bug the +moment TODO-0173 ships. Track as a follow-up rather than folding into 0173 so +each TODO has one job. diff --git a/docs/spec/todos/TODO-0175.md b/docs/spec/todos/TODO-0175.md index f25b1a8..c51f384 100644 --- a/docs/spec/todos/TODO-0175.md +++ b/docs/spec/todos/TODO-0175.md @@ -12,33 +12,31 @@ blocks: [] ## Problem -`grep -rn "TODO-[0-9]" crates/mdvs/src crates/mdvs/examples` returns -~40 hits across ~15 files: TODO-0007, TODO-0097, TODO-0149, TODO-0155, -TODO-0162, TODO-0016, TODO-0159, TODO-0101 and others. They appear in -production code, examples, test docstrings, and module-level docs. +`grep -rn "TODO-[0-9]" crates/mdvs/src crates/mdvs/examples` returns ~40 hits +across ~15 files: TODO-0007, TODO-0097, TODO-0149, TODO-0155, TODO-0162, +TODO-0016, TODO-0159, TODO-0101 and others. They appear in production code, +examples, test docstrings, and module-level docs. -The project rule (CLAUDE.md + memory): code comments document -present-day behavior, not history. Task references rot the moment the -TODO closes/renames/subsumes and turn the code into a changelog. -Historical context belongs in commit messages, PR bodies, and the spec -files themselves. +The project rule (CLAUDE.md + memory): code comments document present-day +behavior, not history. Task references rot the moment the TODO +closes/renames/subsumes and turn the code into a changelog. Historical context +belongs in commit messages, PR bodies, and the spec files themselves. -These refs are accreted technical debt from prior waves and predate -the rule being enforced. +These refs are accreted technical debt from prior waves and predate the rule +being enforced. ## Goal Edit every file containing a `TODO-NNNN` reference. For each comment: -- If the technical content is useful (a hidden constraint, a - workaround, an invariant), **keep the comment, drop the TODO ref.** - Example: `// Per TODO-0162 step 1 spike: native TOML Date values come - through Pod as strings.` → `// Native TOML Date values come through - Pod as strings.` -- If the comment is purely "this was added in TODO-X" with no other - content, **delete it.** -- For doc comments (`///` / `//!`), apply the same rule but be extra - careful — they appear in published API docs. +- If the technical content is useful (a hidden constraint, a workaround, an + invariant), **keep the comment, drop the TODO ref.** Example: + `// Per TODO-0162 step 1 spike: native TOML Date values come through Pod as strings.` + → `// Native TOML Date values come through Pod as strings.` +- If the comment is purely "this was added in TODO-X" with no other content, + **delete it.** +- For doc comments (`///` / `//!`), apply the same rule but be extra careful — + they appear in published API docs. ## Out of scope @@ -48,13 +46,13 @@ Edit every file containing a `TODO-NNNN` reference. For each comment: ## Verification -- `grep -rn "TODO-[0-9]" crates/mdvs/src crates/mdvs/examples` returns - zero hits. +- `grep -rn "TODO-[0-9]" crates/mdvs/src crates/mdvs/examples` returns zero + hits. - `cargo test` + `cargo clippy --all-targets` clean. -- Hand-spot-check 5 random comments for "still reads sensibly" after - the ref is stripped. +- Hand-spot-check 5 random comments for "still reads sensibly" after the ref is + stripped. ## Impact -Low priority — code keeps working — but ships a more professional -codebase before promotion. Estimated 1–2 hours of careful editing. +Low priority — code keeps working — but ships a more professional codebase +before promotion. Estimated 1–2 hours of careful editing. diff --git a/docs/spec/todos/TODO-0176.md b/docs/spec/todos/TODO-0176.md index 8925ae3..c6d2868 100644 --- a/docs/spec/todos/TODO-0176.md +++ b/docs/spec/todos/TODO-0176.md @@ -12,166 +12,154 @@ blocks: [] ## Problem -A three-agent audit (book, docs/spec, README + code-doc) of v0.7.0 -surfaced ~10 critical + ~15 medium findings across ~20 files. Two -themes: - -- The validate + Lance-incremental-write changes are described nowhere - user- or contributor-facing yet — book, spec, and module-level - rustdoc all still describe the pre-rewrite single-write-path flow. -- An older Lance migration (when storage moved off Parquet) left - stale "files.parquet" / "chunks.parquet" / "SipHash" wording across - `crates/mdvs/src/index/` doc-comments. Predates v0.7.0; surfaced - during this sweep so worth fixing in the same pass. +A three-agent audit (book, docs/spec, README + code-doc) of v0.7.0 surfaced ~10 +critical + ~15 medium findings across ~20 files. Two themes: + +- The validate + Lance-incremental-write changes are described nowhere user- or + contributor-facing yet — book, spec, and module-level rustdoc all still + describe the pre-rewrite single-write-path flow. +- An older Lance migration (when storage moved off Parquet) left stale + "files.parquet" / "chunks.parquet" / "SipHash" wording across + `crates/mdvs/src/index/` doc-comments. Predates v0.7.0; surfaced during this + sweep so worth fixing in the same pass. ## Findings by theme ### A. Write-path docs (TODO-0173 shipped, not yet documented) -Everywhere describes `write_index` as a single overwrite + index- -rebuild path. Needs the three-way decision (skip when nothing to -persist, full overwrite when `full_rebuild`, incremental -delete+add+`replace_schema_metadata`+`optimize` for the delta case) -plus the `WriteIndex: Skipped` step appearing in verbose telemetry. +Everywhere describes `write_index` as a single overwrite + index- rebuild path. +Needs the three-way decision (skip when nothing to persist, full overwrite when +`full_rebuild`, incremental delete+add+`replace_schema_metadata`+`optimize` for +the delta case) plus the `WriteIndex: Skipped` step appearing in verbose +telemetry. -- `book/src/commands/build.md:30, 37, 42-54, 95, 137-146` — pipeline - step list, "incremental builds" section, verbose-output sample. +- `book/src/commands/build.md:30, 37, 42-54, 95, 137-146` — pipeline step list, + "incremental builds" section, verbose-output sample. - `book/src/commands/search.md:37` — auto-build framing. - `book/src/concepts/search.md:72` — storage / incremental story. -- `book/src/recipes/obsidian.md:141, 145` — "fast when nothing - changed" undersells; tested-size figure conservative. -- `docs/spec/architecture.md:62, 175, 384-396` — build-pipeline step - 10, LanceBackend method list, Incremental Build section. -- `docs/spec/storage.md:5, 130-138, 148, 101` — write-strategy - description, LanceBackend key-methods list, schema-metadata write - path. +- `book/src/recipes/obsidian.md:141, 145` — "fast when nothing changed" + undersells; tested-size figure conservative. +- `docs/spec/architecture.md:62, 175, 384-396` — build-pipeline step 10, + LanceBackend method list, Incremental Build section. +- `docs/spec/storage.md:5, 130-138, 148, 101` — write-strategy description, + LanceBackend key-methods list, schema-metadata write path. - `docs/spec/commands/build.md:7, 13-17, 22, 30` — `pub fn build_core` - visibility, `ClassifyData.removed_file_ids` field, Write step - description, the "Model skip" bullet (now also "Write skip"). -- `crates/mdvs/src/index/backend.rs:87` — `Backend::write_index` has - no rustdoc; add a summary noting it's the full-rebuild path and - pointing at `write_index_incremental` for the delta path. -- `crates/mdvs/src/cmd/build.rs:186, 288, 942` — `run()` rustdoc says - "Parquet files"; `build_core` summary doesn't list the three write - paths; `detect_config_changes` says "parquet metadata". + visibility, `ClassifyData.removed_file_ids` field, Write step description, the + "Model skip" bullet (now also "Write skip"). +- `crates/mdvs/src/index/backend.rs:87` — `Backend::write_index` has no rustdoc; + add a summary noting it's the full-rebuild path and pointing at + `write_index_incremental` for the delta path. +- `crates/mdvs/src/cmd/build.rs:186, 288, 942` — `run()` rustdoc says "Parquet + files"; `build_core` summary doesn't list the three write paths; + `detect_config_changes` says "parquet metadata". ### B. Validation determinism + `FieldMeta` (TODO-0172 shipped) -The new stable-ordering contract on violation output and the -`FieldMeta` precomputation step are nowhere yet. - -- `book/src/concepts/validation.md:3` — calls validation - "deterministic" without stating the sort key. -- `book/src/commands/check.md` — no mention of stable violation - ordering (sorted by `(field, kind, rule)`, files within each - violation sorted by `path`). -- `docs/spec/commands/check.md:12, 15, 16, 17` — Build-validators - step missing `build_field_metas`; per-field-values step missing - the `is_valid` fast path; path-scoping step should reference the - precomputed GlobSets; "sorts alphabetically" in Collect step is - imprecise — should describe the exact sort keys. -- `docs/spec/architecture.md:188, 209-218` — `ViolationKind` variant - list is in the wrong order vs. declaration and doesn't note that - declaration order is the `Ord` sort key; Validation Pipeline section - doesn't describe `FieldMeta` precomputation or the `is_valid` fast - path. +The new stable-ordering contract on violation output and the `FieldMeta` +precomputation step are nowhere yet. + +- `book/src/concepts/validation.md:3` — calls validation "deterministic" without + stating the sort key. +- `book/src/commands/check.md` — no mention of stable violation ordering (sorted + by `(field, kind, rule)`, files within each violation sorted by `path`). +- `docs/spec/commands/check.md:12, 15, 16, 17` — Build-validators step missing + `build_field_metas`; per-field-values step missing the `is_valid` fast path; + path-scoping step should reference the precomputed GlobSets; "sorts + alphabetically" in Collect step is imprecise — should describe the exact sort + keys. +- `docs/spec/architecture.md:188, 209-218` — `ViolationKind` variant list is in + the wrong order vs. declaration and doesn't note that declaration order is the + `Ord` sort key; Validation Pipeline section doesn't describe `FieldMeta` + precomputation or the `is_valid` fast path. - `docs/spec/shared.md:74-82, 97` — `ViolationKind` snippet missing `PartialOrd, Ord` from the derive list; no statement of the deterministic-output contract on `Vec`. ### C. Pre-existing drift surfaced by the audit (not v0.7.0) -- **Date type recipe**: `book/src/recipes/obsidian.md:64` claims "No - Date type yet — dates are stored as strings." Date / DateTime - shipped (TODO-0007, 2026-05-14). -- **Configuration example**: `book/src/configuration.md:408-422` - example `[scan]` block omits `frontmatter_format = "auto"`. The - example_kb mdvs.toml now writes it explicitly. -- **Parquet→Lance terminology sweep**, all in - `crates/mdvs/src/index/`: +- **Date type recipe**: `book/src/recipes/obsidian.md:64` claims "No Date type + yet — dates are stored as strings." Date / DateTime shipped (TODO-0007, + 2026-05-14). +- **Configuration example**: `book/src/configuration.md:408-422` example + `[scan]` block omits `frontmatter_format = "auto"`. The example_kb mdvs.toml + now writes it explicitly. +- **Parquet→Lance terminology sweep**, all in `crates/mdvs/src/index/`: - `index/mod.rs:3, 9` — "Parquet, future LanceDB" and "Parquet I/O". - - `index/storage.rs:20-42` — per-constant doc-comments saying - "column in files.parquet" / "column in chunks.parquet" (~10 - COL_* constants). + - `index/storage.rs:20-42` — per-constant doc-comments saying "column in + files.parquet" / "column in chunks.parquet" (~10 COL\_\* constants). - `index/storage.rs:49, 63, 82, 99, 119, 140-143, 301` — `FileRow`, `ChunkRow`, `BuildMetadata`, `to_hash_map`, `from_hash_map`, `build_files_batch` all mention parquet. -- **Hash algorithm name**: `crates/mdvs/src/index/storage.rs:57` - says `content_hash` is "SipHash". It's `xxh3`. (The audit also - flagged the scope claim "excluding frontmatter" as wrong — - actually it's right: `content_hash(&f.content)` hashes the - parsed body. TODO-0174 covers fixing the scope itself; here - only fix the algorithm name.) +- **Hash algorithm name**: `crates/mdvs/src/index/storage.rs:57` says + `content_hash` is "SipHash". It's `xxh3`. (The audit also flagged the scope + claim "excluding frontmatter" as wrong — actually it's right: + `content_hash(&f.content)` hashes the parsed body. TODO-0174 covers fixing the + scope itself; here only fix the algorithm name.) ### D. Defensive auto-update / auto-build framing -Every page that mentions `[X].auto_update` reads as if the user -might routinely opt out for performance. After v0.7.0 the only -remaining reasons are determinism (CI) or airgapped operation. +Every page that mentions `[X].auto_update` reads as if the user might routinely +opt out for performance. After v0.7.0 the only remaining reasons are determinism +(CI) or airgapped operation. - `book/src/configuration.md:94-96, 143-146, 162-164` - `book/src/commands/check.md:25` - `book/src/commands/build.md:30` - `book/src/commands/search.md:37` -Tighten each to make determinism the sole motivation; optionally -add a line that the default chain is cheap on unchanged corpora. +Tighten each to make determinism the sole motivation; optionally add a line that +the default chain is cheap on unchanged corpora. ### E. README + AGENTS.md - `README.md:226-227` — "Incremental builds — only changed files are - re-embedded" describes embed-side only; v0.7.0 also made the write - itself incremental. "Auto pipeline … one command does everything" - undersells what's now mdvs's biggest UX win at scale. -- `AGENTS.md:56` — `src/index/` blurb mentions `write_index` only; - doesn't list `write_index_incremental` as a peer method. + re-embedded" describes embed-side only; v0.7.0 also made the write itself + incremental. "Auto pipeline … one command does everything" undersells what's + now mdvs's biggest UX win at scale. +- `AGENTS.md:56` — `src/index/` blurb mentions `write_index` only; doesn't list + `write_index_incremental` as a peer method. ### F. New examples not referenced -`docs/spec/architecture.md` module map doesn't mention the two new -profiling examples: +`docs/spec/architecture.md` module map doesn't mention the two new profiling +examples: -- `crates/mdvs/examples/profile_pipeline.rs` — phase-by-phase wall- - clock harness; drives `build_core` directly and reads `StepEntry` - elapsed timings. -- `crates/mdvs/examples/probe_lance_incremental.rs` — exploratory - LanceDB API probe (copies an index to tempdir, exercises delete / - add / `replace_schema_metadata` / `optimize` with cost numbers). +- `crates/mdvs/examples/profile_pipeline.rs` — phase-by-phase wall- clock + harness; drives `build_core` directly and reads `StepEntry` elapsed timings. +- `crates/mdvs/examples/probe_lance_incremental.rs` — exploratory LanceDB API + probe (copies an index to tempdir, exercises delete / add / + `replace_schema_metadata` / `optimize` with cost numbers). ## Proposed PR shape Two PRs read cleaner than one: -1. **v0.7.0 docs refresh** — Groups A, B, D, E, F. The - write-paths story, the validation determinism contract, the - auto-* framing tightening, the README + AGENTS lines, and the - examples reference. -2. **Parquet→Lance + drift sweep** — Group C. Long-standing - pre-v0.7.0 cleanup; separable. About a dozen `parquet` / - `SipHash` / single-line fixes confined to `crates/mdvs/src/ - index/` plus the `obsidian.md:64` Date claim and +1. **v0.7.0 docs refresh** — Groups A, B, D, E, F. The write-paths story, the + validation determinism contract, the auto-\* framing tightening, the README + + AGENTS lines, and the examples reference. +2. **Parquet→Lance + drift sweep** — Group C. Long-standing pre-v0.7.0 cleanup; + separable. About a dozen `parquet` / `SipHash` / single-line fixes confined + to `crates/mdvs/src/ index/` plus the `obsidian.md:64` Date claim and `configuration.md:408-422` example fix. -One combined PR is also fine if the user prefers — about 20 files -to touch, ~2-3 hours of careful editing. +One combined PR is also fine if the user prefers — about 20 files to touch, ~2-3 +hours of careful editing. ## Verification - `book/` builds (`mdbook build book`) without warnings. -- `cargo doc --no-deps` succeeds — module-level and item-level - doc-comments are syntactically valid Rust. -- Spot-render the verbose `mdvs build` output on `example_kb` to - confirm the `Write index: Skipped` step text we describe matches - what the CLI actually prints. -- Grep `crates/mdvs/src/index/` for `parquet` / `SipHash` post- - sweep → no hits. +- `cargo doc --no-deps` succeeds — module-level and item-level doc-comments are + syntactically valid Rust. +- Spot-render the verbose `mdvs build` output on `example_kb` to confirm the + `Write index: Skipped` step text we describe matches what the CLI actually + prints. +- Grep `crates/mdvs/src/index/` for `parquet` / `SipHash` post- sweep → no hits. - No behavior change is implied; no new tests needed. ## Out of scope -- TODO-0174 (`content_hash` should cover frontmatter — separate - follow-up). -- TODO-0175 (stripping in-code `TODO-NNNN` references — separate - sweep; touching the same files but a different lens). -- New documentation features (a dedicated "Performance" page, a - changelog renderer, etc.) — refresh existing text only. +- TODO-0174 (`content_hash` should cover frontmatter — separate follow-up). +- TODO-0175 (stripping in-code `TODO-NNNN` references — separate sweep; touching + the same files but a different lens). +- New documentation features (a dedicated "Performance" page, a changelog + renderer, etc.) — refresh existing text only. diff --git a/docs/spec/todos/TODO-0177.md b/docs/spec/todos/TODO-0177.md index 63332fc..34ff156 100644 --- a/docs/spec/todos/TODO-0177.md +++ b/docs/spec/todos/TODO-0177.md @@ -15,157 +15,143 @@ files_updated: ## Resolution -Shipped on `docs/readme-rewrite`. The four-audience listing -(Obsidian / Zettelkasten / docs-as-code / wikis) at the top of the -README is gone; the headline stays as a description of the tool; -the agent-curated-KB framing enters naturally in the Validate -section and in a new dedicated "Calling mdvs from an agent" -section showing `--output json | jq` patterns. +Shipped on `docs/readme-rewrite`. The four-audience listing (Obsidian / +Zettelkasten / docs-as-code / wikis) at the top of the README is gone; the +headline stays as a description of the tool; the agent-curated-KB framing enters +naturally in the Validate section and in a new dedicated "Calling mdvs from an +agent" section showing `--output json | jq` patterns. ### What actually changed -- **Audience listing dropped.** No paragraph below the headline - pitching audience — the headline describes what the tool is, - full stop. Audience emerges naturally from the body. -- **Validate section** now closes with one paragraph noting that - validation is "especially useful when an LLM agent is doing the - writing" — drift caught before it compounds. No standalone - agent-pitch section above the example. -- **New "Calling mdvs from an agent" section** with three - `--output json` / `jq` examples: filter violations by kind, - query by metadata + meaning, export the JSON Schema. -- **`--where` examples** use bare frontmatter names (`status = '...'`, - not `data.status = '...'`). mdvs's translator prefixes `data.` - under the hood; the book uses bare names everywhere. -- **Search JSON shape** verified against actual `mdvs search - --output json` output: top-level keys are `{query, hits, model_name, - limit}`, hit field is `filename` (not `file`). Initially wrote - `.hits[].file`, which was wrong; corrected to `.hits[].filename`. -- **`mdvs check --output json` shape** verified: `{files_checked, - new_fields, violations}` (flat, not wrapped). `.violations[]` +- **Audience listing dropped.** No paragraph below the headline pitching + audience — the headline describes what the tool is, full stop. Audience + emerges naturally from the body. +- **Validate section** now closes with one paragraph noting that validation is + "especially useful when an LLM agent is doing the writing" — drift caught + before it compounds. No standalone agent-pitch section above the example. +- **New "Calling mdvs from an agent" section** with three `--output json` / `jq` + examples: filter violations by kind, query by metadata + meaning, export the + JSON Schema. +- **`--where` examples** use bare frontmatter names (`status = '...'`, not + `data.status = '...'`). mdvs's translator prefixes `data.` under the hood; the + book uses bare names everywhere. +- **Search JSON shape** verified against actual `mdvs search --output json` + output: top-level keys are `{query, hits, model_name, limit}`, hit field is + `filename` (not `file`). Initially wrote `.hits[].file`, which was wrong; + corrected to `.hits[].filename`. +- **`mdvs check --output json` shape** verified: + `{files_checked, new_fields, violations}` (flat, not wrapped). `.violations[]` works. - **CWD pattern.** Changed `mdvs init notes/` style examples to - `cd notes/ && mdvs init` — matches how the tool is actually used - (from inside the directory you're managing). -- **`skill` command** added to the Commands table (new in TODO-0185, - was missing from the table). -- **RAG-alternative signal** lives as one phrase in the - Hybrid-search Features bullet ("no GPU, no API keys, no vector-DB - cluster — everything runs in-process"). Not a section, not a - headline claim, just a defining attribute among others. -- **Trailing "longer story of how this tool came together" - link clause** dropped from the Documentation section. The - mdBook origin page that would have backed that clause was - consciously rejected — see "What was decided NOT to do." + `cd notes/ && mdvs init` — matches how the tool is actually used (from inside + the directory you're managing). +- **`skill` command** added to the Commands table (new in TODO-0185, was missing + from the table). +- **RAG-alternative signal** lives as one phrase in the Hybrid-search Features + bullet ("no GPU, no API keys, no vector-DB cluster — everything runs + in-process"). Not a section, not a headline claim, just a defining attribute + among others. +- **Trailing "longer story of how this tool came together" link clause** dropped + from the Documentation section. The mdBook origin page that would have backed + that clause was consciously rejected — see "What was decided NOT to do." ### What was decided NOT to do -- **No mdBook origin / inspirations page.** Discussed at length on - 2026-06-07. Convention for technical documentation books - (ripgrep, fzf, bat, pandoc, etc.) is that origin stories live in - blog posts, not in reference docs. Putting it in the mdBook - would create tone clash and the LLM-fabricated-personal-essay - problem. If the user wants the origin story public, they write - it as a blog post on their terms; the URL gets added to the - README footer if/when it exists. -- **No Karpathy reference in the README.** Decided to mention "the - typical RAG stack" generically as the contrast point, in our own - voice. Karpathy + the convergent-design framing are project - context (saved as a memory `origin_story.md`) but don't appear - in user-facing copy. -- **No tone fix to the mdBook Introduction.** The opener has a - similar "Not a document database. A database for documents." - antithesis line as the README's old one, but the user closed the - TODO without changing it; it can be revisited under TODO-0176 - (full book drift sweep) or whenever. -- **No agent-harness recipe page.** That's TODO-0187 territory, - post-launch. +- **No mdBook origin / inspirations page.** Discussed at length on 2026-06-07. + Convention for technical documentation books (ripgrep, fzf, bat, pandoc, etc.) + is that origin stories live in blog posts, not in reference docs. Putting it + in the mdBook would create tone clash and the LLM-fabricated-personal-essay + problem. If the user wants the origin story public, they write it as a blog + post on their terms; the URL gets added to the README footer if/when it + exists. +- **No Karpathy reference in the README.** Decided to mention "the typical RAG + stack" generically as the contrast point, in our own voice. Karpathy + the + convergent-design framing are project context (saved as a memory + `origin_story.md`) but don't appear in user-facing copy. +- **No tone fix to the mdBook Introduction.** The opener has a similar "Not a + document database. A database for documents." antithesis line as the README's + old one, but the user closed the TODO without changing it; it can be revisited + under TODO-0176 (full book drift sweep) or whenever. +- **No agent-harness recipe page.** That's TODO-0187 territory, post-launch. ### Verification -- `mdvs init`, `mdvs check`, `mdvs search`, `mdvs export-jsonschema` - on a scratch vault all behave as the README claims. -- Exit codes (`0` / `1` / `2`) confirmed via `echo $?` after - injecting a violation. -- The README's two output snippets (check + search rendered tables) - use the actual mdvs output format (trimmed widths to fit). -- The init output section in the README dropped (gif covers it; a - paraphrased text format would have been misleading). -- Length: 209 lines (was 250). Target was 150-180; overshoots due - to the rendered output snippets the user wanted kept for - visceral appeal. +- `mdvs init`, `mdvs check`, `mdvs search`, `mdvs export-jsonschema` on a + scratch vault all behave as the README claims. +- Exit codes (`0` / `1` / `2`) confirmed via `echo $?` after injecting a + violation. +- The README's two output snippets (check + search rendered tables) use the + actual mdvs output format (trimmed widths to fit). +- The init output section in the README dropped (gif covers it; a paraphrased + text format would have been misleading). +- Length: 209 lines (was 250). Target was 150-180; overshoots due to the + rendered output snippets the user wanted kept for visceral appeal. ## Problem ## Problem -An outside review (2026-06-04, action item 1) flagged the README -rewrite as the highest-leverage pre-launch change. Today the -README pitches four audiences — Obsidian users, Zettelkasten, -docs-as-code, personal-wiki — and lands with none. The reviewer's -proposed pivot to "docs-as-code teams running Hugo/MkDocs/Astro" is -a reasonable second-best but not the right call: it would frame mdvs -as a frontmatter linter that happens to do search, when the actual -shape (LanceDB hybrid search + Model2Vec embeddings + multi-format -frontmatter + schema inference) is better suited to a deeper -audience. +An outside review (2026-06-04, action item 1) flagged the README rewrite as the +highest-leverage pre-launch change. Today the README pitches four audiences — +Obsidian users, Zettelkasten, docs-as-code, personal-wiki — and lands with none. +The reviewer's proposed pivot to "docs-as-code teams running Hugo/MkDocs/Astro" +is a reasonable second-best but not the right call: it would frame mdvs as a +frontmatter linter that happens to do search, when the actual shape (LanceDB +hybrid search + Model2Vec embeddings + multi-format frontmatter + schema +inference) is better suited to a deeper audience. ## Audience -**People maintaining a typed markdown knowledge base — especially -LLM-curated ones in the pattern Andrej Karpathy described (April -2026): a `raw/` + `wiki/` + `outputs/` folder layout where an LLM -agent reads raw materials, writes organized wiki entries, and -queries them later. Also Obsidian / Logseq power users with large -vaults who want their notes to be more than a flat file pile.** +**People maintaining a typed markdown knowledge base — especially LLM-curated +ones in the pattern Andrej Karpathy described (April 2026): a `raw/` + `wiki/` + +`outputs/` folder layout where an LLM agent reads raw materials, writes +organized wiki entries, and queries them later. Also Obsidian / Logseq power +users with large vaults who want their notes to be more than a flat file pile.** These users: -- Are technical enough for `cargo install` (they already are calling - tools from Claude Code / Codex / similar). -- Treat the KB as a database they want to query, not files they want - to grep. -- Care that the schema stays consistent as the corpus grows — without - it, agents drift, the corpus rots, and search recall degrades. -- Want both human-callable CLI (interactive) and agent-callable - surface (`--output json`, deterministic exit codes, SQL `--where`). - -The reviewer's docs-as-code pitch is a **secondary** audience worth -mentioning at the bottom — frontmatter linting in CI is a real use -case but it's not the headline. + +- Are technical enough for `cargo install` (they already are calling tools from + Claude Code / Codex / similar). +- Treat the KB as a database they want to query, not files they want to grep. +- Care that the schema stays consistent as the corpus grows — without it, agents + drift, the corpus rots, and search recall degrades. +- Want both human-callable CLI (interactive) and agent-callable surface + (`--output json`, deterministic exit codes, SQL `--where`). + +The reviewer's docs-as-code pitch is a **secondary** audience worth mentioning +at the bottom — frontmatter linting in CI is a real use case but it's not the +headline. ## Approach Rewrite the README to lead with the KB-as-typed-database story: -1. **One-line pitch:** "mdvs turns a markdown directory into a typed - database with schema validation and hybrid search — the data layer - under an LLM-curated knowledge base." (Wording tbd; that's the - shape.) -2. **Lead example: an LLM-driven KB workflow.** A short asciinema or - code block showing `mdvs check`, `mdvs search "..."`, and a - `--where data.tags = '...'` query that an agent would naturally - use. Skip the "watch me search a vault" framing — show why the - schema + the search together matter. -3. **Multi-format frontmatter callout.** Move it up. Real - differentiator nobody in the niche ships, and important to the - KB-curator audience whose notes come from heterogeneous sources - (Hugo `+++`, Obsidian `---`, Hugo JSON, etc.). -4. **Search section** with the three modes + the `--where` syntax — - not the lead, but a substantive second half. -5. **Schema inference + JSON Schema export** — present as "your KB - gets a typed contract" rather than as a generic dev-tool feature. -6. **Drop the four-audience listing** at the top. Replace with one - audience and a short "also useful for: docs-as-code CI / personal - wikis / Zettelkasten" footer at the bottom. -7. **Keep the benchmark link** but don't lead with the latency - numbers. Mention them once, link to `docs/benchmarks/report.md`. +1. **One-line pitch:** "mdvs turns a markdown directory into a typed database + with schema validation and hybrid search — the data layer under an + LLM-curated knowledge base." (Wording tbd; that's the shape.) +2. **Lead example: an LLM-driven KB workflow.** A short asciinema or code block + showing `mdvs check`, `mdvs search "..."`, and a `--where data.tags = '...'` + query that an agent would naturally use. Skip the "watch me search a vault" + framing — show why the schema + the search together matter. +3. **Multi-format frontmatter callout.** Move it up. Real differentiator nobody + in the niche ships, and important to the KB-curator audience whose notes come + from heterogeneous sources (Hugo `+++`, Obsidian `---`, Hugo JSON, etc.). +4. **Search section** with the three modes + the `--where` syntax — not the + lead, but a substantive second half. +5. **Schema inference + JSON Schema export** — present as "your KB gets a typed + contract" rather than as a generic dev-tool feature. +6. **Drop the four-audience listing** at the top. Replace with one audience and + a short "also useful for: docs-as-code CI / personal wikis / Zettelkasten" + footer at the bottom. +7. **Keep the benchmark link** but don't lead with the latency numbers. Mention + them once, link to `docs/benchmarks/report.md`. ## Verification -- A Karpathy-style KB user reads the README and says "this is the - layer I've been hand-rolling." -- A casual Obsidian user reads it and says "I get what this is for, - even if I'd rather use a plugin." +- A Karpathy-style KB user reads the README and says "this is the layer I've + been hand-rolling." +- A casual Obsidian user reads it and says "I get what this is for, even if I'd + rather use a plugin." - Word count down, not up. - One audience, one problem, one demo path through the page. @@ -174,14 +160,13 @@ Rewrite the README to lead with the KB-as-typed-database story: - The mdBook content (separate refresh). - The new asciinema demo aimed at this audience — TODO-0178. - Any code change; this is a framing change. -- Adding agent-specific features (a JSON-RPC server, an MCP shim, - etc.) — `--output json` already exists; emphasize what we have, - don't add surface area pre-launch. +- Adding agent-specific features (a JSON-RPC server, an MCP shim, etc.) — + `--output json` already exists; emphasize what we have, don't add surface area + pre-launch. ## Impact -The single most leveraged change before publicly amplifying the -project. Without it, the next round of feedback will be "I don't -know who this is for." With it, the project meets a real underserved -niche head-on (LLM-curated KBs need data infrastructure, and almost -nobody is building it). +The single most leveraged change before publicly amplifying the project. Without +it, the next round of feedback will be "I don't know who this is for." With it, +the project meets a real underserved niche head-on (LLM-curated KBs need data +infrastructure, and almost nobody is building it). diff --git a/docs/spec/todos/TODO-0178.md b/docs/spec/todos/TODO-0178.md index 87ef236..2dcd0b4 100644 --- a/docs/spec/todos/TODO-0178.md +++ b/docs/spec/todos/TODO-0178.md @@ -12,64 +12,62 @@ blocks: [] ## Problem -The current `assets/demo.cast` shows "watch me search a vault" — the -tool working, not why someone needs it. An outside review (2026-06-04, -action item 2) flagged the existing demo as the second-largest gap -between "real artifact" and "something a stranger would adopt." +The current `assets/demo.cast` shows "watch me search a vault" — the tool +working, not why someone needs it. An outside review (2026-06-04, action item 2) +flagged the existing demo as the second-largest gap between "real artifact" and +"something a stranger would adopt." ## Audience -Same as TODO-0177: someone maintaining a typed markdown knowledge -base, especially in the Karpathy LLM-curated pattern. The demo must -make the **why** visceral in under 90 seconds. +Same as TODO-0177: someone maintaining a typed markdown knowledge base, +especially in the Karpathy LLM-curated pattern. The demo must make the **why** +visceral in under 90 seconds. ## Approach -Build a new demo script that walks through a realistic KB-curator -flow rather than a search highlight reel: +Build a new demo script that walks through a realistic KB-curator flow rather +than a search highlight reel: -1. Start in a folder of markdown notes — preferably with the - Karpathy `raw/` + `wiki/` shape so the structure is recognizable. - Could be the existing `demo_kb/` repurposed, or a new fixture. -2. **`mdvs init`** — show the schema being inferred from the - frontmatter. The user (or "the agent") didn't write the schema; - mdvs did. -3. **`mdvs check`** with a deliberately broken file — show the - typed-violation table. This is what catches agent drift. +1. Start in a folder of markdown notes — preferably with the Karpathy `raw/` + + `wiki/` shape so the structure is recognizable. Could be the existing + `demo_kb/` repurposed, or a new fixture. +2. **`mdvs init`** — show the schema being inferred from the frontmatter. The + user (or "the agent") didn't write the schema; mdvs did. +3. **`mdvs check`** with a deliberately broken file — show the typed-violation + table. This is what catches agent drift. 4. **Fix the file, `mdvs check` clean.** Quick. -5. **`mdvs search "..." --where data.tags = '...'`** — show that the - typed schema makes filtered semantic search meaningful, not just - a wall of cosine matches. -6. **`mdvs search "..." --output json | jq …`** — show the agent- - callable surface. One line in the script narrating "this is what - an LLM agent would call." That's the actual headline. +5. **`mdvs search "..." --where data.tags = '...'`** — show that the typed + schema makes filtered semantic search meaningful, not just a wall of cosine + matches. +6. **`mdvs search "..." --output json | jq …`** — show the agent- callable + surface. One line in the script narrating "this is what an LLM agent would + call." That's the actual headline. 7. Keep under ~90 seconds total; trim aggressively. -Technical approach mirrors the existing demo infra (`assets/demo.py` -is PEP 723 / `uv run`, drives bash via pexpect inside -`asciinema rec`). Output: a new `assets/demo.cast` + rendered -`.gif`; old assets archived rather than deleted (the search-vault -demo still works, just no longer the primary). +Technical approach mirrors the existing demo infra (`assets/demo.py` is PEP 723 +/ `uv run`, drives bash via pexpect inside `asciinema rec`). Output: a new +`assets/demo.cast` + rendered `.gif`; old assets archived rather than deleted +(the search-vault demo still works, just no longer the primary). ## Verification -- A new viewer who doesn't already know mdvs watches the gif and can - articulate, unprompted: "it makes a markdown folder act like a - typed database with search." -- The agent-callable beat lands — viewer registers `--output json` - as the thing that makes mdvs usable from Claude Code / Codex. +- A new viewer who doesn't already know mdvs watches the gif and can articulate, + unprompted: "it makes a markdown folder act like a typed database with + search." +- The agent-callable beat lands — viewer registers `--output json` as the thing + that makes mdvs usable from Claude Code / Codex. - Length under 90 seconds. ## Out of scope - Building an actual LLM-agent demo that requires API keys (a script - + voiceover narrating "this is the call an agent makes" suffices). + - voiceover narrating "this is the call an agent makes" suffices). - Embedding the demo in the mdBook (separate concern). -- Replacing demo_kb wholesale unless the existing one's content - doesn't fit the new script. +- Replacing demo_kb wholesale unless the existing one's content doesn't fit the + new script. ## Impact -Pairs with the README rewrite (TODO-0177) — the README sets the -pitch, the demo proves it. Together they're the pre-amplify pair -that determines whether the project gets adopted or dismissed. +Pairs with the README rewrite (TODO-0177) — the README sets the pitch, the demo +proves it. Together they're the pre-amplify pair that determines whether the +project gets adopted or dismissed. diff --git a/docs/spec/todos/TODO-0179.md b/docs/spec/todos/TODO-0179.md index 764fe45..2a1c7f1 100644 --- a/docs/spec/todos/TODO-0179.md +++ b/docs/spec/todos/TODO-0179.md @@ -30,241 +30,221 @@ files_updated: ## Resolution -Shipped as 4 commits on `refactor/module-split`, one per wave. All -4 over-budget production files (cmd/check, cmd/build, -schema/json_schema, index/backend) now have their largest piece -under ~600 production lines. The reviewer's "looks LLM-written" -signal is gone — no monoliths left in the tree. +Shipped as 4 commits on `refactor/module-split`, one per wave. All 4 over-budget +production files (cmd/check, cmd/build, schema/json_schema, index/backend) now +have their largest piece under ~600 production lines. The reviewer's "looks +LLM-written" signal is gone — no monoliths left in the tree. ### Per-wave outcome -| Wave | Original | After split (production lines) | -|------|---------:|---| -| 1 — `cmd/build.rs` | 1,012 | mod.rs 676 (orchestrator: `run` + `build_core`), classify.rs 117, config_mutate.rs 185, embed.rs 55, write.rs 90 | -| 2 — `schema/json_schema.rs` | 826 | mod.rs 58 (re-exports + `is_intermediate_object`), to_canonical.rs 288, from_canonical.rs 347, validate.rs 170 | -| 3 — `index/backend.rs` | 925 | mod.rs 440 (Backend enum + write paths + shared helpers), read.rs 145, search.rs 385 | -| 4 — `cmd/check.rs` | 918 | mod.rs 323 (`run` + `resolve_check_config`), validate.rs 328, collect.rs 241, field_meta.rs 176 | +| Wave | Original | After split (production lines) | +| --------------------------- | -------: | ---------------------------------------------------------------------------------------------------------------- | +| 1 — `cmd/build.rs` | 1,012 | mod.rs 676 (orchestrator: `run` + `build_core`), classify.rs 117, config_mutate.rs 185, embed.rs 55, write.rs 90 | +| 2 — `schema/json_schema.rs` | 826 | mod.rs 58 (re-exports + `is_intermediate_object`), to_canonical.rs 288, from_canonical.rs 347, validate.rs 170 | +| 3 — `index/backend.rs` | 925 | mod.rs 440 (Backend enum + write paths + shared helpers), read.rs 145, search.rs 385 | +| 4 — `cmd/check.rs` | 918 | mod.rs 323 (`run` + `resolve_check_config`), validate.rs 328, collect.rs 241, field_meta.rs 176 | -Each wave is its own commit (`a69f127`, `6b5dd7b`, `1234087`, -`f185b4f`). `git diff --find-renames` cleanly attributes the -original-file → `/mod.rs` move on each wave (50-76% rename -similarity). +Each wave is its own commit (`a69f127`, `6b5dd7b`, `1234087`, `f185b4f`). +`git diff --find-renames` cleanly attributes the original-file → `/mod.rs` +move on each wave (50-76% rename similarity). ### One accepted overage -`cmd/build/mod.rs` landed at 676 production lines — 76 over the -~600 target. Discussed at the end of Wave 1 and accepted: further -splitting `build_core` would fragment the pipeline narrative -(scan → infer → validate → load model → classify → embed → write) -across files for a marginal LOC win. The orchestrator is the -orchestrator; the reviewer's concern was about 2,000-line monoliths, +`cmd/build/mod.rs` landed at 676 production lines — 76 over the ~600 target. +Discussed at the end of Wave 1 and accepted: further splitting `build_core` +would fragment the pipeline narrative (scan → infer → validate → load model → +classify → embed → write) across files for a marginal LOC win. The orchestrator +is the orchestrator; the reviewer's concern was about 2,000-line monoliths, which is now gone. ### Visibility approach (uniform across waves) - Items used cross-sub-module: `pub(super)`. -- Items re-exported to the rest of the crate: `pub(crate)` (sometimes - `pub` when an example reaches them). +- Items re-exported to the rest of the crate: `pub(crate)` (sometimes `pub` when + an example reaches them). - Public API surface unchanged at the parent module path — `crate::cmd::build::*`, `crate::schema::json_schema::*`, - `crate::index::backend::*`, `crate::cmd::check::*` all see the - same items they did before. + `crate::index::backend::*`, `crate::cmd::check::*` all see the same items they + did before. ### Verification - `cargo build` and `cargo build --features testing-mocks` clean. -- `cargo test --features testing-mocks` — 804 + 12 ignored, same - as pre-refactor. +- `cargo test --features testing-mocks` — 804 + 12 ignored, same as + pre-refactor. - `cargo clippy --all-targets --features testing-mocks -- -D warnings` clean. - `cargo fmt --check` clean. -- No behavior regression on `example_kb` (the example suite that - exercises every code path stayed green). +- No behavior regression on `example_kb` (the example suite that exercises every + code path stayed green). ### Test classification -Per the LOC metric reframe (production code only, co-located unit -tests don't count), the existing `#[cfg(test)] mod tests {}` blocks -stayed in the parent `mod.rs` for each refactor. The classify -unit tests (in cmd/build) and the normalize_revision unit test -(also cmd/build) moved with their function to the sub-module, as -those tested specific internals. Everything else stayed parent-mod -to avoid churn. +Per the LOC metric reframe (production code only, co-located unit tests don't +count), the existing `#[cfg(test)] mod tests {}` blocks stayed in the parent +`mod.rs` for each refactor. The classify unit tests (in cmd/build) and the +normalize_revision unit test (also cmd/build) moved with their function to the +sub-module, as those tested specific internals. Everything else stayed +parent-mod to avoid churn. ## Problem -Per an outside review (2026-06-04, action item 3), several modules -in `crates/mdvs/src/` are big enough to read as a classic LLM tell. -The accretion is a real thing each module did honestly — each wave -added one more concern to `build_core`, each Wave-B step added a -translator branch to `json_schema.rs`, each Lance API call landed -in `backend.rs`. Result: long files that a skeptical reader skims, -sees the line count, and dismisses. +Per an outside review (2026-06-04, action item 3), several modules in +`crates/mdvs/src/` are big enough to read as a classic LLM tell. The accretion +is a real thing each module did honestly — each wave added one more concern to +`build_core`, each Wave-B step added a translator branch to `json_schema.rs`, +each Lance API call landed in `backend.rs`. Result: long files that a skeptical +reader skims, sees the line count, and dismisses. ## LOC metric — production lines only -The threshold ("no file over ~600 lines") applies to **production -code**, not the total line count. Co-located unit tests -(`#[cfg(test)] mod tests { ... }` at the bottom of the file) are -**not counted** against the budget. This matches the [Rust Book's -test-organization recommendation](https://doc.rust-lang.org/book/ch11-03-test-organization.html): -unit tests belong in the same file as the code they test, because -that's the only way to assert on private state. Extracting them to -satisfy a line-count metric would force tests through the public -interface and lose the private-API access that makes co-located -unit tests valuable. +The threshold ("no file over ~600 lines") applies to **production code**, not +the total line count. Co-located unit tests (`#[cfg(test)] mod tests { ... }` at +the bottom of the file) are **not counted** against the budget. This matches the +[Rust Book's test-organization recommendation](https://doc.rust-lang.org/book/ch11-03-test-organization.html): +unit tests belong in the same file as the code they test, because that's the +only way to assert on private state. Extracting them to satisfy a line-count +metric would force tests through the public interface and lose the private-API +access that makes co-located unit tests valuable. -A Rust-fluent reader mentally subtracts the `#[cfg(test)]` block -when sizing a file. The metric we enforce should match. +A Rust-fluent reader mentally subtracts the `#[cfg(test)]` block when sizing a +file. The metric we enforce should match. ## Current inventory Audit on 2026-06-06 (`wc -l` minus the `#[cfg(test)]` boundary): -| File | Total | Production | Over budget? | -|---|---|---|---| -| `cmd/check.rs` | 2,904 | **918** | Yes | -| `cmd/build.rs` | 2,308 | **1,012** | Yes | -| `schema/json_schema.rs` | 1,972 | **826** | Yes | -| `index/backend.rs` | 1,522 | **925** | Yes | -| `schema/config.rs` | 1,983 | 582 | Under | -| `index/storage.rs` | 1,450 | 531 | Under | -| `cmd/search.rs` | 1,245 | 339 | Under | -| `discover/infer/mod.rs` | 1,198 | 267 | Under | -| `cmd/update.rs` | 975 | <600 (verify) | Under | -| `discover/scan.rs` | 972 | <600 (verify) | Under | -| `discover/field_type.rs` | 944 | <600 (verify) | Under | -| `preprocess.rs` | 779 | <600 (verify) | Under | -| `schema/shared.rs` | 773 | <600 (verify) | Under | -| `discover/infer/types.rs` | 698 | <600 (verify) | Under | - -Four files need structural splits. Everything else is over 600 -total only because of co-located tests — leave alone. +| File | Total | Production | Over budget? | +| ------------------------- | ----- | ------------- | ------------ | +| `cmd/check.rs` | 2,904 | **918** | Yes | +| `cmd/build.rs` | 2,308 | **1,012** | Yes | +| `schema/json_schema.rs` | 1,972 | **826** | Yes | +| `index/backend.rs` | 1,522 | **925** | Yes | +| `schema/config.rs` | 1,983 | 582 | Under | +| `index/storage.rs` | 1,450 | 531 | Under | +| `cmd/search.rs` | 1,245 | 339 | Under | +| `discover/infer/mod.rs` | 1,198 | 267 | Under | +| `cmd/update.rs` | 975 | <600 (verify) | Under | +| `discover/scan.rs` | 972 | <600 (verify) | Under | +| `discover/field_type.rs` | 944 | <600 (verify) | Under | +| `preprocess.rs` | 779 | <600 (verify) | Under | +| `schema/shared.rs` | 773 | <600 (verify) | Under | +| `discover/infer/types.rs` | 698 | <600 (verify) | Under | + +Four files need structural splits. Everything else is over 600 total only +because of co-located tests — leave alone. ## Approach -Mechanical refactor only — **no behavior change**, the public API -surface stays identical. When a file splits into sub-modules, the -existing tests **move with the code they test** — each new -sub-module keeps its co-located `#[cfg(test)] mod tests {}` for -the functions it now contains. Integration-flavored tests (the ones -that exercise the whole pipeline end-to-end) stay in the parent -`mod.rs`. +Mechanical refactor only — **no behavior change**, the public API surface stays +identical. When a file splits into sub-modules, the existing tests **move with +the code they test** — each new sub-module keeps its co-located +`#[cfg(test)] mod tests {}` for the functions it now contains. +Integration-flavored tests (the ones that exercise the whole pipeline +end-to-end) stay in the parent `mod.rs`. -The work is four independent waves, one per file. Each wave is a -single commit so the diff is easy to review and `git diff --find- -renames=50%` shows clear pure-move ratios. +The work is four independent waves, one per file. Each wave is a single commit +so the diff is easy to review and `git diff --find- renames=50%` shows clear +pure-move ratios. ### Wave 1 — `cmd/build.rs` (1,012 production → target ~3–5 files under ~600 each) -Convert `cmd/build.rs` → `cmd/build/mod.rs` plus the sub-modules -below. Each sub-module owns its own `#[cfg(test)] mod tests {}` -covering only the functions it now contains. +Convert `cmd/build.rs` → `cmd/build/mod.rs` plus the sub-modules below. Each +sub-module owns its own `#[cfg(test)] mod tests {}` covering only the functions +it now contains. - **`cmd/build/classify.rs`** — `ClassifyData`, `ClassifyOutcome`, - `classify_files` function plus the per-file content-hash logic. - Carries the tests for `classify_*` paths. -- **`cmd/build/embed.rs`** — `embed_file`, the per-chunk embed - loop, and helpers that own the embedder invocation. Carries - embed-specific tests. -- **`cmd/build/write.rs`** — the `write_index` / `write_index_ - incremental` dispatch glue (NB: the backend-side incremental - code itself stays in `index/backend.rs`). Carries + `classify_files` function plus the per-file content-hash logic. Carries the + tests for `classify_*` paths. +- **`cmd/build/embed.rs`** — `embed_file`, the per-chunk embed loop, and helpers + that own the embedder invocation. Carries embed-specific tests. +- **`cmd/build/write.rs`** — the `write_index` / `write_index_ incremental` + dispatch glue (NB: the backend-side incremental code itself stays in + `index/backend.rs`). Carries `second_build_skips_write_index_when_nothing_changed` and `third_build_persists_new_file_via_incremental_path`. -- **`cmd/build/config_mutate.rs`** — `mutate_config`, - `normalize_revision`, `detect_config_changes`. Carries - `manual_config_change_detected`, `set_model_*`, and - `set_chunk_size_*` tests. -- **`cmd/build/mod.rs`** — orchestrator `build_core`, `run`, the - `pub use` re-exports for backward compatibility, plus the - integration-flavored end-to-end tests (`end_to_end`, - `dimension_mismatch`, `incremental_*`, `force_full_rebuild`, - `build_aborts_on_*`, `build_succeeds_with_*`). Should land - around 400 lines including these tests. +- **`cmd/build/config_mutate.rs`** — `mutate_config`, `normalize_revision`, + `detect_config_changes`. Carries `manual_config_change_detected`, + `set_model_*`, and `set_chunk_size_*` tests. +- **`cmd/build/mod.rs`** — orchestrator `build_core`, `run`, the `pub use` + re-exports for backward compatibility, plus the integration-flavored + end-to-end tests (`end_to_end`, `dimension_mismatch`, `incremental_*`, + `force_full_rebuild`, `build_aborts_on_*`, `build_succeeds_with_*`). Should + land around 400 lines including these tests. ### Wave 2 — `schema/json_schema.rs` (826 production → ~3 files) Convert `schema/json_schema.rs` → `schema/json_schema/mod.rs` plus: -- **`schema/json_schema/to_canonical.rs`** — `dsl_to_canonical` + - the per-field-type → JSON-Schema converters + scope helpers. - Tests for the DSL-to-canonical direction. -- **`schema/json_schema/from_canonical.rs`** — `canonical_to_dsl` + - the reverse converters + the strict pattern matches against the - shapes `dsl_to_canonical` produces. Tests for the reverse - direction. -- **`schema/json_schema/validate.rs`** — `validate_mdvs_schema` - gate + the curated allow / deny keyword lists + the explanatory - rejection messages. Tests for the gate. +- **`schema/json_schema/to_canonical.rs`** — `dsl_to_canonical` + the + per-field-type → JSON-Schema converters + scope helpers. Tests for the + DSL-to-canonical direction. +- **`schema/json_schema/from_canonical.rs`** — `canonical_to_dsl` + the reverse + converters + the strict pattern matches against the shapes `dsl_to_canonical` + produces. Tests for the reverse direction. +- **`schema/json_schema/validate.rs`** — `validate_mdvs_schema` gate + the + curated allow / deny keyword lists + the explanatory rejection messages. Tests + for the gate. - **`schema/json_schema/mod.rs`** — `compute_schema_hash`, - `extract_leaf_schemas`, the small shared helpers + the public - re-exports + the round-trip tests that exercise both directions - together. + `extract_leaf_schemas`, the small shared helpers + the public re-exports + the + round-trip tests that exercise both directions together. ### Wave 3 — `index/backend.rs` (925 production → ~2–3 files) Convert `index/backend.rs` → `index/backend/mod.rs` plus: -- **`index/backend/search.rs`** — `SearchMode` matching, - `nearest_to` / FTS / hybrid query construction, `--where` - translation (including the `lazy_regex` patterns from - TODO-0180), the `over-fetch-and-dedupe` Rust-side ranking +- **`index/backend/search.rs`** — `SearchMode` matching, `nearest_to` / FTS / + hybrid query construction, `--where` translation (including the `lazy_regex` + patterns from TODO-0180), the `over-fetch-and-dedupe` Rust-side ranking helpers. Tests for search dispatch + `--where` translation. -- **`index/backend/read.rs`** — `read_file_index`, `read_chunk_ - rows`, `read_metadata`, `embedding_dimension`, `stats`. Tests - for the reader path. -- **`index/backend/mod.rs`** — `Backend` enum + `LanceBackend` - struct + `write_index` / `write_index_incremental` (the actual - write paths stay together because they share the `build_indexes` - helper). Connection/db-dir/table-dir helpers also live here. - Tests for the write paths and the metadata round-trip. +- **`index/backend/read.rs`** — `read_file_index`, `read_chunk_ rows`, + `read_metadata`, `embedding_dimension`, `stats`. Tests for the reader path. +- **`index/backend/mod.rs`** — `Backend` enum + `LanceBackend` struct + + `write_index` / `write_index_incremental` (the actual write paths stay + together because they share the `build_indexes` helper). + Connection/db-dir/table-dir helpers also live here. Tests for the write paths + and the metadata round-trip. ### Wave 4 — `cmd/check.rs` (918 production — was not in the original TODO) This file grew after TODO-0172's `FieldMeta` precomputation + the -deterministic-output contract landed. It's now the worst offender -of the four. Splits along the seams the cheap-wins refactor -already introduced: - -- **`cmd/check/field_meta.rs`** — `FieldMeta` struct, - `build_field_metas` (the per-field precompute pass: - `GlobSet` compile, `FieldType::try_from`, `Validator` - compile). Tests for the precompute pipeline. -- **`cmd/check/validate.rs`** — the `validate` function itself - (the per-document scan + per-field check + `is_valid` fast - path), plus the dotted-path navigation helpers - (`navigate_dotted`). Tests for the per-field validators. -- **`cmd/check/collect.rs`** — `collect_violations` plus the - `ViolationKind` → `FieldViolation` mapping + the deterministic - sort. Tests for the sort contract. -- **`cmd/check/mod.rs`** — `run`, the `--jsonschema` override - plumbing, the top-level orchestration that wires precompute → - validate → collect, plus the end-to-end tests - (`check_violations_on_*`, `check_passes_on_*`). Re-exports for - backward compatibility. - -Seam confirmation needed: read `cmd/check.rs` once before wave 4 -starts to make sure the boundaries above match the actual -function layout (the structure is fresh after TODO-0172 so this -is mostly a sanity check). +deterministic-output contract landed. It's now the worst offender of the four. +Splits along the seams the cheap-wins refactor already introduced: + +- **`cmd/check/field_meta.rs`** — `FieldMeta` struct, `build_field_metas` (the + per-field precompute pass: `GlobSet` compile, `FieldType::try_from`, + `Validator` compile). Tests for the precompute pipeline. +- **`cmd/check/validate.rs`** — the `validate` function itself (the per-document + scan + per-field check + `is_valid` fast path), plus the dotted-path + navigation helpers (`navigate_dotted`). Tests for the per-field validators. +- **`cmd/check/collect.rs`** — `collect_violations` plus the `ViolationKind` → + `FieldViolation` mapping + the deterministic sort. Tests for the sort + contract. +- **`cmd/check/mod.rs`** — `run`, the `--jsonschema` override plumbing, the + top-level orchestration that wires precompute → validate → collect, plus the + end-to-end tests (`check_violations_on_*`, `check_passes_on_*`). Re-exports + for backward compatibility. + +Seam confirmation needed: read `cmd/check.rs` once before wave 4 starts to make +sure the boundaries above match the actual function layout (the structure is +fresh after TODO-0172 so this is mostly a sanity check). ## Verification (per wave) After each wave's commit: -- `cargo test --features testing-mocks` — full suite green, only - test changes are file moves + `use super::*` adjustments. +- `cargo test --features testing-mocks` — full suite green, only test changes + are file moves + `use super::*` adjustments. - `cargo clippy --all-targets --features testing-mocks -- -D warnings` clean. - `cargo fmt --check` clean. -- `git diff --stat --find-renames=50%` shows clear pure-move - ratios (anything below 50% is suspicious — flag and explain). +- `git diff --stat --find-renames=50%` shows clear pure-move ratios (anything + below 50% is suspicious — flag and explain). - Spot-check `cargo doc --no-deps` produces the same public items: - `mdvs::cmd::build::{build_core, run}` - `mdvs::index::backend::{Backend, LanceBackend, SearchMode, SearchHit}` - `mdvs::schema::json_schema::{dsl_to_canonical, canonical_to_dsl, validate_mdvs_schema, compute_schema_hash, extract_leaf_schemas}` - `mdvs::cmd::check::run` -- No behavior regression on `example_kb` (`mdvs check / build / - search` unchanged). +- No behavior regression on `example_kb` (`mdvs check / build / search` + unchanged). - Each split file's production-LOC under ~600. ## Out of scope @@ -273,16 +253,15 @@ After each wave's commit: - Removing or merging functionality. - Doc updates beyond what the move forces (mostly intra-doc `crate::cmd::build::...` paths). -- Trimming or rewriting the function bodies themselves — that's - fair-game in a separate cleanup pass, not here. This TODO is - about file shape, not code quality. -- Files under 600 production lines, even if total (with tests) is - over 600. See [LOC metric — production lines only](#loc-metric--production-lines-only). +- Trimming or rewriting the function bodies themselves — that's fair-game in a + separate cleanup pass, not here. This TODO is about file shape, not code + quality. +- Files under 600 production lines, even if total (with tests) is over 600. See + [LOC metric — production lines only](#loc-metric--production-lines-only). ## Impact -Per the reviewer, "the single highest-impact `doesn't look LLM- -written` change." Multiple smaller files signal the kind of -human-rotated work a reader expects to see; long monoliths signal -the kind they don't. Counting production lines (not co-located -tests) keeps the metric honest by Rust's own convention. +Per the reviewer, "the single highest-impact `doesn't look LLM- written` +change." Multiple smaller files signal the kind of human-rotated work a reader +expects to see; long monoliths signal the kind they don't. Counting production +lines (not co-located tests) keeps the metric honest by Rust's own convention. diff --git a/docs/spec/todos/TODO-0180.md b/docs/spec/todos/TODO-0180.md index a77a731..bc69fe2 100644 --- a/docs/spec/todos/TODO-0180.md +++ b/docs/spec/todos/TODO-0180.md @@ -21,181 +21,185 @@ files_updated: ## Resolution -Audit ran via four parallel agents covering the whole non-test -surface of `crates/mdvs/src/` and `crates/tomljson/src/`. Findings -were much sparser than the TODO anticipated: **7 sites total**, none -of them user-input-driven. The user-facing panic concern that -motivated the TODO ("the first power user feeds weird input and -hits a panic") was already addressed — every input path is `Result`- -typed end-to-end. What remained were 7 invariant-guarded panic- +Audit ran via four parallel agents covering the whole non-test surface of +`crates/mdvs/src/` and `crates/tomljson/src/`. Findings were much sparser than +the TODO anticipated: **7 sites total**, none of them user-input-driven. The +user-facing panic concern that motivated the TODO ("the first power user feeds +weird input and hits a panic") was already addressed — every input path is +`Result`- typed end-to-end. What remained were 7 invariant-guarded panic- emitters — all reachable only via mdvs / tomljson programmer bugs. Inventory of the 7 sites: -- `tomljson/src/de.rs:55-57` — `Number::from_f64(*f).expect("finite float must convert")` after explicit NaN/Inf checks two lines up. -- `tomljson/src/ser.rs:74-78` — `n.as_u64().expect("checked is_u64")` after `n.is_u64()` guard above. -- `tomljson/src/ser.rs:144-152` — `unreachable!("u64 overflow rejected in assert_encodable")` trusting the upstream precheck. -- `tomljson/src/ser.rs:243-263` — `_ => unreachable!("is_array_of_tables guarantees Object")` matching elements of a Vec already filtered by the predicate. -- `crates/mdvs/src/render.rs:76` — `&rows[row_idx][0]` double-indexing into caller-supplied row indices. -- `crates/mdvs/src/index/backend.rs:768` — `Regex::new("...").expect("valid literal regex")` on a compile-time string literal. -- `crates/mdvs/src/index/backend.rs:769` — `Regex::new("...").expect("valid ident regex")` on a compile-time string literal. +- `tomljson/src/de.rs:55-57` — + `Number::from_f64(*f).expect("finite float must convert")` after explicit + NaN/Inf checks two lines up. +- `tomljson/src/ser.rs:74-78` — `n.as_u64().expect("checked is_u64")` after + `n.is_u64()` guard above. +- `tomljson/src/ser.rs:144-152` — + `unreachable!("u64 overflow rejected in assert_encodable")` trusting the + upstream precheck. +- `tomljson/src/ser.rs:243-263` — + `_ => unreachable!("is_array_of_tables guarantees Object")` matching elements + of a Vec already filtered by the predicate. +- `crates/mdvs/src/render.rs:76` — `&rows[row_idx][0]` double-indexing into + caller-supplied row indices. +- `crates/mdvs/src/index/backend.rs:768` — + `Regex::new("...").expect("valid literal regex")` on a compile-time string + literal. +- `crates/mdvs/src/index/backend.rs:769` — + `Regex::new("...").expect("valid ident regex")` on a compile-time string + literal. ### What we shipped -Discussion with the user surfaced that "provably-infallible" was -too strong a category — every one of the 7 sites was actually a -"currently-guarded by a local invariant" risk, fragile to a future -refactor that drops the guard. Five of the seven were rewritten to -eliminate the trust-the-distant-guard pattern entirely; the other -two switched to compile-time-verified regex via `lazy_regex`. +Discussion with the user surfaced that "provably-infallible" was too strong a +category — every one of the 7 sites was actually a "currently-guarded by a local +invariant" risk, fragile to a future refactor that drops the guard. Five of the +seven were rewritten to eliminate the trust-the-distant-guard pattern entirely; +the other two switched to compile-time-verified regex via `lazy_regex`. **Five refactors:** 1. **`tomljson/de.rs`** — replaced explicit NaN/Inf branches + - `.expect("finite float must convert")` with a single `match - serde_json::Number::from_f64(*f) { Some(n) => ..., None => ... }`. - The conversion is now the validation; no separate `.is_nan()` / - `.is_infinite()` checks to drift apart from the conversion call. - -2. **`tomljson/ser.rs:74-82`** (`assert_encodable`, Number branch) — - replaced `if n.is_u64() && expect("checked is_u64")` with - `if let Some(value) = n.as_u64().filter(|_| n.as_i64().is_none())`. - One operation, no separate guard. - -3. **`tomljson/ser.rs:140-160`** (`write_inline`, Number branch) — - replaced `unreachable!("u64 overflow rejected in assert_encodable")` - with explicit `else if let Some(value) = n.as_u64() { return - Err(IntegerOutOfRange { ... }) }` plus a defensive `Err` for the - (impossible) "no accessor matched" case. If a future refactor - bypasses `assert_encodable`, the user sees a clean recoverable - error instead of a panic. - -4. **`tomljson/ser.rs:207-263`** (`write_table`, `sub_aots`) — changed - the storage type of `sub_aots` from `Vec<(&str, &Vec)>` to - `Vec<(&str, Vec<&serde_json::Map<...>>)>`. Typed maps are - extracted via `filter_map` at collection time; the AOT emission - loop iterates typed maps directly. The `_ => unreachable!()` arm - is gone — the type system carries the proof. - -5. **`crates/mdvs/src/render.rs:76`** — replaced `&rows[row_idx][0]` - with `rows.get(row_idx).and_then(|r| r.first()).filter(|s| !s.is_empty())` - and a `continue` if any link fails. Bad caller indices now skip - the panel instead of crashing rendering. + `.expect("finite float must convert")` with a single + `match serde_json::Number::from_f64(*f) { Some(n) => ..., None => ... }`. The + conversion is now the validation; no separate `.is_nan()` / `.is_infinite()` + checks to drift apart from the conversion call. + +2. **`tomljson/ser.rs:74-82`** (`assert_encodable`, Number branch) — replaced + `if n.is_u64() && expect("checked is_u64")` with + `if let Some(value) = n.as_u64().filter(|_| n.as_i64().is_none())`. One + operation, no separate guard. + +3. **`tomljson/ser.rs:140-160`** (`write_inline`, Number branch) — replaced + `unreachable!("u64 overflow rejected in assert_encodable")` with explicit + `else if let Some(value) = n.as_u64() { return Err(IntegerOutOfRange { ... }) }` + plus a defensive `Err` for the (impossible) "no accessor matched" case. If a + future refactor bypasses `assert_encodable`, the user sees a clean + recoverable error instead of a panic. + +4. **`tomljson/ser.rs:207-263`** (`write_table`, `sub_aots`) — changed the + storage type of `sub_aots` from `Vec<(&str, &Vec)>` to + `Vec<(&str, Vec<&serde_json::Map<...>>)>`. Typed maps are extracted via + `filter_map` at collection time; the AOT emission loop iterates typed maps + directly. The `_ => unreachable!()` arm is gone — the type system carries the + proof. + +5. **`crates/mdvs/src/render.rs:76`** — replaced `&rows[row_idx][0]` with + `rows.get(row_idx).and_then(|r| r.first()).filter(|s| !s.is_empty())` and a + `continue` if any link fails. Bad caller indices now skip the panel instead + of crashing rendering. **Two compile-time-verified regex sites:** -6+7. `index/backend.rs:768-769` — switched to `lazy_regex::regex!`. -The macro parses the pattern at `cargo build` time, so a syntax -error becomes a compile error rather than a runtime panic. Runtime -construction is `LazyLock`-style, amortized to once per -process (slightly faster than the original `Regex::new` per call). -Pulled in the small `lazy-regex = "3"` dep. `cargo audit` / -`cargo deny check` both clean — no new transitive runtime deps +6+7. `index/backend.rs:768-769` — switched to `lazy_regex::regex!`. The macro +parses the pattern at `cargo build` time, so a syntax error becomes a compile +error rather than a runtime panic. Runtime construction is +`LazyLock`-style, amortized to once per process (slightly faster than the +original `Regex::new` per call). Pulled in the small `lazy-regex = "3"` dep. +`cargo audit` / `cargo deny check` both clean — no new transitive runtime deps beyond what mdvs already pulls. ### Regression gate -Added `#![cfg_attr(not(test), warn(clippy::unwrap_used, -clippy::expect_used, clippy::panic))]` to the crate roots of both -`mdvs` and `tomljson`. The gate fires only on non-test code. Any -future PR that introduces a new `.unwrap()` / `.expect()` / `panic!` -in production code fails CI unless the contributor adds a local -`#[allow(...)]` with a justifying comment. The `cargo clippy ---all-targets --features testing-mocks -- -D warnings` step in CI +Added +`#![cfg_attr(not(test), warn(clippy::unwrap_used, clippy::expect_used, clippy::panic))]` +to the crate roots of both `mdvs` and `tomljson`. The gate fires only on +non-test code. Any future PR that introduces a new `.unwrap()` / `.expect()` / +`panic!` in production code fails CI unless the contributor adds a local +`#[allow(...)]` with a justifying comment. The +`cargo clippy --all-targets --features testing-mocks -- -D warnings` step in CI carries the enforcement. ### Verification - `cargo build` and `cargo build --features testing-mocks` clean. -- `cargo test --features testing-mocks` — 804 pass, 12 ignored (slow-lane real-model tests from TODO-0184). +- `cargo test --features testing-mocks` — 804 pass, 12 ignored (slow-lane + real-model tests from TODO-0184). - `cargo clippy --all-targets --features testing-mocks -- -D warnings` clean. - `cargo fmt --check` clean. -- `cargo audit` — no new findings beyond the 4 pre-existing unmaintained-crate warnings from upstream deps (`bincode`, `encoding`, `number_prefix`, `paste` — all from `lance-tokenizer`, `gray_matter`, `indicatif`, `datafusion`). +- `cargo audit` — no new findings beyond the 4 pre-existing unmaintained-crate + warnings from upstream deps (`bincode`, `encoding`, `number_prefix`, `paste` — + all from `lance-tokenizer`, `gray_matter`, `indicatif`, `datafusion`). - `cargo deny check` — `advisories ok, bans ok, licenses ok, sources ok`. ### Out of scope (intentionally deferred) -- Stress-testing mdvs against malformed YAML / TOML / JSON / NUL - bytes / malformed globs / Lance-missing-metadata isn't in this - PR's scope — those paths are already `Result`-typed per the - audit, so no test gap was uncovered. Worth doing later as a - fuzz pass, but not gated by TODO-0180. +- Stress-testing mdvs against malformed YAML / TOML / JSON / NUL bytes / + malformed globs / Lance-missing-metadata isn't in this PR's scope — those + paths are already `Result`-typed per the audit, so no test gap was uncovered. + Worth doing later as a fuzz pass, but not gated by TODO-0180. ## Problem -An outside review (2026-06-04, action item 4) flagged that "the -first power user who feeds weird input will hit a `panic!` or -`unwrap` and the goodwill is gone." The existing rule -in memory (`feedback_no_panic_in_prod.md`) already says no -`.expect()` / `.unwrap()` / `panic!` / `todo!` / `unreachable!` -outside tests — but the rule isn't enforced systematically. A -focused audit is needed before public amplification. +An outside review (2026-06-04, action item 4) flagged that "the first power user +who feeds weird input will hit a `panic!` or `unwrap` and the goodwill is gone." +The existing rule in memory (`feedback_no_panic_in_prod.md`) already says no +`.expect()` / `.unwrap()` / `panic!` / `todo!` / `unreachable!` outside tests — +but the rule isn't enforced systematically. A focused audit is needed before +public amplification. ## Approach -1. **Grep first.** Inside `crates/mdvs/src/` (and `crates/tomljson/ - src/`), find every occurrence outside `#[cfg(test)]` of: +1. **Grep first.** Inside `crates/mdvs/src/` (and `crates/tomljson/ src/`), find + every occurrence outside `#[cfg(test)]` of: - `.unwrap()` - `.expect(...)` - `panic!(...)` - `todo!(...)`, `unimplemented!(...)`, `unreachable!(...)` - Quick rough count: `rg -n '\.unwrap\(\)|\.expect\(|panic!\(|todo!\(|unimplemented!\(|unreachable!\(' crates/mdvs/src crates/tomljson/src --type rust` + Quick rough count: + `rg -n '\.unwrap\(\)|\.expect\(|panic!\(|todo!\(|unimplemented!\(|unreachable!\(' crates/mdvs/src crates/tomljson/src --type rust` 2. **Categorize each hit** into one of: - - **Provably-infallible** — context guarantees the variant. Replace - with `expect("…why this can't fail…")` carrying the invariant. - - **Recoverable** — input could legitimately cause this. Convert to - `?` returning an `anyhow::Error` (or a specific error variant - where the call site cares). - - **Programmer-error** — a state we never expect; reachable only - via mdvs bugs. Keep as `unreachable!` or `expect` with a clear - "internal invariant violated:" prefix. - - **Genuinely safe** — `unwrap` on a `Option` produced two lines - above with a known-`Some` source. These are fine if the - proximity makes the invariant obvious; convert to `expect` with - a one-word reason regardless, for the next reader. - -3. **Test-only modules** are out of scope; skip everything under - `#[cfg(test)]` and the `tests/` directories. - -4. **Document the invariants** as you go — every remaining `expect` - should have a message that explains *why* the call can't fail. - This is the difference between "we hand-wave at the user" and - "we wrote down the invariant." - -5. **Add a clippy gate** if practical (`#![warn(clippy::unwrap_used)]` - on the production crate root, scoped to non-test code) so the - audit stays clean. + - **Provably-infallible** — context guarantees the variant. Replace with + `expect("…why this can't fail…")` carrying the invariant. + - **Recoverable** — input could legitimately cause this. Convert to `?` + returning an `anyhow::Error` (or a specific error variant where the call + site cares). + - **Programmer-error** — a state we never expect; reachable only via mdvs + bugs. Keep as `unreachable!` or `expect` with a clear "internal invariant + violated:" prefix. + - **Genuinely safe** — `unwrap` on a `Option` produced two lines above with a + known-`Some` source. These are fine if the proximity makes the invariant + obvious; convert to `expect` with a one-word reason regardless, for the + next reader. + +3. **Test-only modules** are out of scope; skip everything under `#[cfg(test)]` + and the `tests/` directories. + +4. **Document the invariants** as you go — every remaining `expect` should have + a message that explains _why_ the call can't fail. This is the difference + between "we hand-wave at the user" and "we wrote down the invariant." + +5. **Add a clippy gate** if practical (`#![warn(clippy::unwrap_used)]` on the + production crate root, scoped to non-test code) so the audit stays clean. ## Verification - `rg -n '\.unwrap\(\)' crates/mdvs/src crates/tomljson/src --type rust -g '!**/tests/**'` - returns zero hits (modulo intentional cases with explanatory - comments). + returns zero hits (modulo intentional cases with explanatory comments). - Every remaining `expect()` has a non-trivial message. -- `cargo clippy --all-targets` clean. If the warn-gate is added, - no new warnings. -- Stress test: feed mdvs malformed YAML, malformed TOML, malformed - JSON, files with NUL bytes, an empty `mdvs.toml`, an - `mdvs.toml` with a malformed glob, a Lance index missing - metadata keys. None should panic; all should produce a clean - `ErrorKind::User` or `Application` step. +- `cargo clippy --all-targets` clean. If the warn-gate is added, no new + warnings. +- Stress test: feed mdvs malformed YAML, malformed TOML, malformed JSON, files + with NUL bytes, an empty `mdvs.toml`, an `mdvs.toml` with a malformed glob, a + Lance index missing metadata keys. None should panic; all should produce a + clean `ErrorKind::User` or `Application` step. ## Out of scope -- Refactoring the error types themselves — that's a bigger surface - change. Use whatever the call site already uses (`anyhow::Result` - in most of `cmd/`, custom enums in `schema/`). +- Refactoring the error types themselves — that's a bigger surface change. Use + whatever the call site already uses (`anyhow::Result` in most of `cmd/`, + custom enums in `schema/`). - Touching test code. -- Examples (`crates/mdvs/examples/`) — these are intentionally - permissive about unwraps for one-off scripts. Optional sweep at - the end, but not blocking. +- Examples (`crates/mdvs/examples/`) — these are intentionally permissive about + unwraps for one-off scripts. Optional sweep at the end, but not blocking. ## Impact -Per the reviewer, this is the single change most likely to -determine whether someone's first try with mdvs survives. A panic -on a real corpus is a launch-blocker for the kind of users -TODO-0177 targets (LLM-curated KBs where the input shape varies -because agents generate it). +Per the reviewer, this is the single change most likely to determine whether +someone's first try with mdvs survives. A panic on a real corpus is a +launch-blocker for the kind of users TODO-0177 targets (LLM-curated KBs where +the input shape varies because agents generate it). diff --git a/docs/spec/todos/TODO-0181.md b/docs/spec/todos/TODO-0181.md index b32abab..efb52b6 100644 --- a/docs/spec/todos/TODO-0181.md +++ b/docs/spec/todos/TODO-0181.md @@ -12,94 +12,89 @@ blocks: [] ## Problem -An outside review (2026-06-04, action item 8) flagged three places -where mdvs ships abstraction scaffolding without the feature behind -it. A reader who finds an empty enum or a -reserved keyword that goes nowhere reads it as -"author over-designed and stopped." The reviewer's call was -"delete it or build it"; we want a middle path: **keep what we -intend to extend, with a written reason; remove what's truly orphan.** +An outside review (2026-06-04, action item 8) flagged three places where mdvs +ships abstraction scaffolding without the feature behind it. A reader who finds +an empty enum or a reserved keyword that goes nowhere reads it as "author +over-designed and stopped." The reviewer's call was "delete it or build it"; we +want a middle path: **keep what we intend to extend, with a written reason; +remove what's truly orphan.** ## Three places ### 1. `ValueStage::Stage1` and `ValueStage::Stage3` (keep + annotate) -`preprocess.rs` declares a three-stage preprocessor pipeline (Stage 1 -= field-name normalization, Stage 2 = per-value coercion, Stage 3 = -per-document). Only Stage 2 has real variants (`coerce_to_string`, -`widen_int_to_float`); Stage 1 and Stage 3 are name-bearing but empty. - -These stay. The pipeline architecture **is** the design — running -preprocessors in three stages with explicit boundaries is the -right shape even when two of the three are empty today. The fix is -to make the intent visible to a reader landing on the file cold. - -Action: add module-level documentation explaining the three-stage -design, why Stage 1 and Stage 3 are currently empty, and the kinds -of preprocessors that would go in each. Targets: snake-case -normalization for field names (Stage 1); cross-field consistency -checks like "if `draft == true` then `published == false`" -(Stage 3). Mention that the lattice is fixed by design and won't -grow beyond three stages — empty doesn't mean half-built; it -means no preprocessor has emerged that wants that slot yet. +`preprocess.rs` declares a three-stage preprocessor pipeline (Stage 1 = +field-name normalization, Stage 2 = per-value coercion, Stage 3 = per-document). +Only Stage 2 has real variants (`coerce_to_string`, `widen_int_to_float`); Stage +1 and Stage 3 are name-bearing but empty. + +These stay. The pipeline architecture **is** the design — running preprocessors +in three stages with explicit boundaries is the right shape even when two of the +three are empty today. The fix is to make the intent visible to a reader landing +on the file cold. + +Action: add module-level documentation explaining the three-stage design, why +Stage 1 and Stage 3 are currently empty, and the kinds of preprocessors that +would go in each. Targets: snake-case normalization for field names (Stage 1); +cross-field consistency checks like "if `draft == true` then +`published == false`" (Stage 3). Mention that the lattice is fixed by design and +won't grow beyond three stages — empty doesn't mean half-built; it means no +preprocessor has emerged that wants that slot yet. ### 2. `x-mdvs.definitions` (investigate, then likely remove) -The canonical JSON Schema emitted by `dsl_to_canonical` reserves -the key `x-mdvs.definitions` for future `$defs` / `$ref` support -(referenceable subschemas to avoid duplicating Array(Object) -shapes). Today the translator never writes to it; the validator -never reads it; nothing in mdvs supports `$defs` end-to-end. It's -pure name-squatting. - -Action: confirm the key is unread + unwritten in the current code -(it should be), then remove the reservation. If we later decide to -ship `$defs`, the name is still available — we don't lose anything -by not pre-declaring it. There's an existing TODO ([TODO-0155 / its -follow-up TODO-0156 on Array(Object) representation](TODO-0156.md)) +The canonical JSON Schema emitted by `dsl_to_canonical` reserves the key +`x-mdvs.definitions` for future `$defs` / `$ref` support (referenceable +subschemas to avoid duplicating Array(Object) shapes). Today the translator +never writes to it; the validator never reads it; nothing in mdvs supports +`$defs` end-to-end. It's pure name-squatting. + +Action: confirm the key is unread + unwritten in the current code (it should +be), then remove the reservation. If we later decide to ship `$defs`, the name +is still available — we don't lose anything by not pre-declaring it. There's an +existing TODO +([TODO-0155 / its follow-up TODO-0156 on Array(Object) representation](TODO-0156.md)) where reusable definitions would naturally re-enter the design. ### 3. Dead invariants in `MdvsToml::validate()` -`MdvsToml::validate()` has nine invariants. Some of them guard -against shapes the inference would never produce after the -grammar tightening in TODO-0155 (e.g. checks that an `Array(Object{ -...})` element shape isn't somewhere it shouldn't be). Those are -fossils: the input that would trigger them no longer exists. +`MdvsToml::validate()` has nine invariants. Some of them guard against shapes +the inference would never produce after the grammar tightening in TODO-0155 +(e.g. checks that an `Array(Object{ ...})` element shape isn't somewhere it +shouldn't be). Those are fossils: the input that would trigger them no longer +exists. -Action: enumerate each invariant; for each, check whether it's -genuinely reachable on input the current code produces or accepts. -Delete the unreachable ones; keep them in their tests by either -removing the test entirely (if the input it constructs is no -longer valid TOML at parse time) or rewriting the test to assert -the parse-time rejection instead. +Action: enumerate each invariant; for each, check whether it's genuinely +reachable on input the current code produces or accepts. Delete the unreachable +ones; keep them in their tests by either removing the test entirely (if the +input it constructs is no longer valid TOML at parse time) or rewriting the test +to assert the parse-time rejection instead. ## Verification -- Stage 1 / Stage 3: a reader hitting `preprocess.rs` cold can tell - in 30 seconds what's there and why it's empty. The empty enums - are documented, not bare. -- `x-mdvs.definitions`: grep for the literal string returns zero - hits in `src/`. Removed from the canonical JSON emitted by - `dsl_to_canonical` and from any tests that asserted on it. -- `MdvsToml::validate()`: the invariant count is documented and - each remaining one has at least one test that constructs reaching - input. Removed invariants have their tests removed or rewritten - to assert at the parse layer. +- Stage 1 / Stage 3: a reader hitting `preprocess.rs` cold can tell in 30 + seconds what's there and why it's empty. The empty enums are documented, not + bare. +- `x-mdvs.definitions`: grep for the literal string returns zero hits in `src/`. + Removed from the canonical JSON emitted by `dsl_to_canonical` and from any + tests that asserted on it. +- `MdvsToml::validate()`: the invariant count is documented and each remaining + one has at least one test that constructs reaching input. Removed invariants + have their tests removed or rewritten to assert at the parse layer. - All existing behavior tests still pass. ## Out of scope -- Building Stage 1 / Stage 3 preprocessors (the headroom is the - point; we keep it explicitly). -- Building `$defs` support (separate design, currently captured by - TODO-0156 if and when it's needed). -- Removing other forms of design headroom in the codebase — only - the three the reviewer named. +- Building Stage 1 / Stage 3 preprocessors (the headroom is the point; we keep + it explicitly). +- Building `$defs` support (separate design, currently captured by TODO-0156 if + and when it's needed). +- Removing other forms of design headroom in the codebase — only the three the + reviewer named. ## Impact -A reader who pokes at the abstraction edges finds either a written -reason ("this is empty because we expect X to land here later") or -nothing ("there's no `definitions` keyword, no orphan placeholder"). -Either way, the "looks over-designed and abandoned" signal is gone. +A reader who pokes at the abstraction edges finds either a written reason ("this +is empty because we expect X to land here later") or nothing ("there's no +`definitions` keyword, no orphan placeholder"). Either way, the "looks +over-designed and abandoned" signal is gone. diff --git a/docs/spec/todos/TODO-0182.md b/docs/spec/todos/TODO-0182.md index 7a2e64b..7b426dd 100644 --- a/docs/spec/todos/TODO-0182.md +++ b/docs/spec/todos/TODO-0182.md @@ -10,79 +10,89 @@ blocks: [] ## Resolution (2026-06-07) -Audited all `docs/spec/` pages (excluding `archive/` and `todos/`). Most pages are current — TODO-0176 had already refreshed `architecture.md`, `storage.md`, `shared.md`, `commands/build.md`, and `commands/check.md`. The remaining content (`inference.md`, `search.md`, `release.md`, the other per-command pages) was accurate. +Audited all `docs/spec/` pages (excluding `archive/` and `todos/`). Most pages +are current — TODO-0176 had already refreshed `architecture.md`, `storage.md`, +`shared.md`, `commands/build.md`, and `commands/check.md`. The remaining content +(`inference.md`, `search.md`, `release.md`, the other per-command pages) was +accurate. Three things didn't pull their weight: -- **`docs/spec/workflows/`** — empty directory, holdover from an earlier folder plan. Deleted. -- **`docs/spec/cocogitto.md`** — 103 lines, mostly redundant with `release.md`'s `cog bump` section. Only inbound link was `release.md:41` "see this for the full guide". Deleted; the reference now points readers directly at the [conventional commits spec](https://www.conventionalcommits.org/), which is the actual upstream reference. -- **`docs/spec/assessments/2026-06-04-outside-review.md`** — single point-in-time review file. Deleted along with the `assessments/` folder. +- **`docs/spec/workflows/`** — empty directory, holdover from an earlier folder + plan. Deleted. +- **`docs/spec/cocogitto.md`** — 103 lines, mostly redundant with `release.md`'s + `cog bump` section. Only inbound link was `release.md:41` "see this for the + full guide". Deleted; the reference now points readers directly at the + [conventional commits spec](https://www.conventionalcommits.org/), which is + the actual upstream reference. +- **`docs/spec/assessments/2026-06-04-outside-review.md`** — single + point-in-time review file. Deleted along with the `assessments/` folder. -Folded in a small drift fix in `docs/spec/search.md`: five references to `index/backend.rs` (singular file) updated to the post-TODO-0179 directory layout (`index/backend/mod.rs`, `index/backend/search.rs`). +Folded in a small drift fix in `docs/spec/search.md`: five references to +`index/backend.rs` (singular file) updated to the post-TODO-0179 directory +layout (`index/backend/mod.rs`, `index/backend/search.rs`). -Active spec surface is now 14 pages (5 top-level + 8 in `commands/` + 1 in `archive/` not counted). `MEMORY.md`'s "Specs Structure" line refreshed to match. +Active spec surface is now 14 pages (5 top-level + 8 in `commands/` + 1 in +`archive/` not counted). `MEMORY.md`'s "Specs Structure" line refreshed to +match. # TODO-0182: Prune stale and useless spec pages ## Problem -The `docs/spec/` directory has pages that don't pull their weight — -content that's superseded by later waves, stubs that never got -fleshed out, or design diaries that were useful in-flight but -aren't a navigable reference today. An outside review (2026-06-04, -action item 6) framed this as "more spec pages than code modules"; -that framing is not the right reason to prune. The right reason is -that stale pages mislead anyone trying to use the spec as a current -map of the codebase. +The `docs/spec/` directory has pages that don't pull their weight — content +that's superseded by later waves, stubs that never got fleshed out, or design +diaries that were useful in-flight but aren't a navigable reference today. An +outside review (2026-06-04, action item 6) framed this as "more spec pages than +code modules"; that framing is not the right reason to prune. The right reason +is that stale pages mislead anyone trying to use the spec as a current map of +the codebase. ## Approach -Per-page audit. For each `.md` under `docs/spec/` (excluding -`todos/`, which is a separate concern — those stay as our working -surface — and `archive/`, which is intentionally point-in-time): - -1. **Is it currently accurate?** Does it describe the code as it - exists at v0.7.0, or does it describe something we did - differently / never built? -2. **Does anyone use it?** Is it referenced from another spec page, - from the book, from a CLAUDE.md / agent skill? If nothing links - it and the content is duplicated elsewhere, it's a candidate. -3. **Is the content load-bearing or vestigial?** Stub pages with - one-paragraph "TODO" placeholders, design diaries that have been - superseded by the shipped commit, multi-page exploration of an - alternative that wasn't taken — all candidates. +Per-page audit. For each `.md` under `docs/spec/` (excluding `todos/`, which is +a separate concern — those stay as our working surface — and `archive/`, which +is intentionally point-in-time): + +1. **Is it currently accurate?** Does it describe the code as it exists at + v0.7.0, or does it describe something we did differently / never built? +2. **Does anyone use it?** Is it referenced from another spec page, from the + book, from a CLAUDE.md / agent skill? If nothing links it and the content is + duplicated elsewhere, it's a candidate. +3. **Is the content load-bearing or vestigial?** Stub pages with one-paragraph + "TODO" placeholders, design diaries that have been superseded by the shipped + commit, multi-page exploration of an alternative that wasn't taken — all + candidates. Disposition for each page: - **Keep as-is** if it's an accurate current reference. -- **Update** if the structure is right but the content is stale (do - this via TODO-0176's docs refresh wave; don't double-up here). -- **Merge** into another page if it's a fragment with no - standalone purpose. -- **Move to `docs/spec/archive/`** if it captured a decision worth - preserving but doesn't belong in the active spec. +- **Update** if the structure is right but the content is stale (do this via + TODO-0176's docs refresh wave; don't double-up here). +- **Merge** into another page if it's a fragment with no standalone purpose. +- **Move to `docs/spec/archive/`** if it captured a decision worth preserving + but doesn't belong in the active spec. - **Delete** if it's a stub / superseded / not worth archiving. ## Verification -- A contributor opens `docs/spec/` cold and the remaining pages all - describe something currently true about the codebase. +- A contributor opens `docs/spec/` cold and the remaining pages all describe + something currently true about the codebase. - No remaining page is a stub or a "TODO: write this section" placeholder. - No remaining page has more than ~10% content that's no longer accurate. - Book pages that link into `docs/spec/` still resolve. -- `MEMORY.md`'s "Specs Structure" line is updated if pages moved - or merged. +- `MEMORY.md`'s "Specs Structure" line is updated if pages moved or merged. ## Out of scope - `docs/spec/todos/` — our working surface; separate from this. - `docs/spec/archive/` — already intentional history. - Rewriting / refactoring page content (that's TODO-0176). -- Reducing the number of spec pages as a goal in itself; the only - reason to remove a page is that it's stale or vestigial. +- Reducing the number of spec pages as a goal in itself; the only reason to + remove a page is that it's stale or vestigial. ## Impact -The spec becomes a navigable current reference instead of a mix of -current + historical + aspirational. Pairs with TODO-0176 (refresh -remaining content) and TODO-0183 (trim CLAUDE.md). +The spec becomes a navigable current reference instead of a mix of current + +historical + aspirational. Pairs with TODO-0176 (refresh remaining content) and +TODO-0183 (trim CLAUDE.md). diff --git a/docs/spec/todos/TODO-0183.md b/docs/spec/todos/TODO-0183.md index 1efb5ed..ccc2a43 100644 --- a/docs/spec/todos/TODO-0183.md +++ b/docs/spec/todos/TODO-0183.md @@ -15,126 +15,118 @@ files_updated: ## Resolution -AGENTS.md (symlinked from CLAUDE.md) trimmed from 140 lines to 52 — -under the original ~50–80 target. Diff: 24 insertions, 112 -deletions. +AGENTS.md (symlinked from CLAUDE.md) trimmed from 140 lines to 52 — under the +original ~50–80 target. Diff: 24 insertions, 112 deletions. ### What survived -- **Project Overview** — one-line elevator pitch + pointers to - `docs/spec/` and `book/`. -- **Git Rules** — branch model, "never push to main", the - literal-"commit" authorization rule, the conventional-commits - reference. These are the agent-behavior contract. -- **Build & Verify** — the cargo commands the agent runs, plus - the `testing-mocks` feature note (project-specific quirk). -- **Architectural Invariants** — six rules that survive refactors: - enum dispatch (no `dyn Trait`), two-layer validation/search, - strict types, mdvs.toml as single source of truth, build - includes check, no interactive prompts. These are real - constraints on future changes. -- **Pointers** — to spec docs, `mdvs --help`, `Cargo.toml` - comments, `.claude/skills/`, and `docs/spec/todos/index.md`. +- **Project Overview** — one-line elevator pitch + pointers to `docs/spec/` and + `book/`. +- **Git Rules** — branch model, "never push to main", the literal-"commit" + authorization rule, the conventional-commits reference. These are the + agent-behavior contract. +- **Build & Verify** — the cargo commands the agent runs, plus the + `testing-mocks` feature note (project-specific quirk). +- **Architectural Invariants** — six rules that survive refactors: enum dispatch + (no `dyn Trait`), two-layer validation/search, strict types, mdvs.toml as + single source of truth, build includes check, no interactive prompts. These + are real constraints on future changes. +- **Pointers** — to spec docs, `mdvs --help`, `Cargo.toml` comments, + `.claude/skills/`, and `docs/spec/todos/index.md`. ### What was cut -- **Module-by-module map** of `src/` directories (~10 lines) — - derivable from a `tree`-style listing or `cargo doc`. +- **Module-by-module map** of `src/` directories (~10 lines) — derivable from a + `tree`-style listing or `cargo doc`. - **Data Pipeline** narrative (1 long paragraph) — belongs in `docs/spec/architecture.md`. -- **Key Design Decisions** bullet list (~20 bullets, most - describing per-feature behavior rather than architectural - invariants) — kept the 6 invariants, moved the implementation - details to spec. -- **Storage / Configuration / Commands / Dependencies** sections — - storage detail belongs in `docs/spec/storage.md`; configuration - in `docs/spec/architecture.md`; commands in `mdvs --help` + - `docs/spec/commands/`; dependency rationale in - `crates/mdvs/Cargo.toml` comments. +- **Key Design Decisions** bullet list (~20 bullets, most describing per-feature + behavior rather than architectural invariants) — kept the 6 invariants, moved + the implementation details to spec. +- **Storage / Configuration / Commands / Dependencies** sections — storage + detail belongs in `docs/spec/storage.md`; configuration in + `docs/spec/architecture.md`; commands in `mdvs --help` + + `docs/spec/commands/`; dependency rationale in `crates/mdvs/Cargo.toml` + comments. ### Where the design history lives now -Per the TODO's verification bullet ("design history moved to -commits / CHANGELOG / archive lives in a durable place, not -lost"), nothing was lost — every wave / TODO mentioned in the old -file is preserved in: - -- Per-commit messages (each wave has its own commit on the branch - it shipped from). -- `docs/spec/todos/index.md` and per-TODO files (the design diary - of every wave: 0007 Date types, 0097 dotted-name flattening, - 0149 jsonschema engine, 0162 multi-format frontmatter, 0172 - validate cheap wins, 0173 incremental Lance writes, etc.). +Per the TODO's verification bullet ("design history moved to commits / CHANGELOG +/ archive lives in a durable place, not lost"), nothing was lost — every wave / +TODO mentioned in the old file is preserved in: + +- Per-commit messages (each wave has its own commit on the branch it shipped + from). +- `docs/spec/todos/index.md` and per-TODO files (the design diary of every wave: + 0007 Date types, 0097 dotted-name flattening, 0149 jsonschema engine, 0162 + multi-format frontmatter, 0172 validate cheap wins, 0173 incremental Lance + writes, etc.). - `docs/spec/archive/` for older specs. -A reader picking up the project cold now hits a 52-line agent- -rules file plus an explicit pointer at `docs/spec/todos/` and -`docs/spec/architecture.md` for the design diary — better than -reading the full diary inline every session. +A reader picking up the project cold now hits a 52-line agent- rules file plus +an explicit pointer at `docs/spec/todos/` and `docs/spec/architecture.md` for +the design diary — better than reading the full diary inline every session. ## Problem -An outside review (2026-06-04, action item 7) flagged CLAUDE.md as -containing design history that should live in commits and CHANGELOG, -not in an agent-instructions file. Today -CLAUDE.md is the single instructions file for every agent that -touches this repo (symlinked from `.cursorrules` etc.), so its -content is read fresh into every agent session — long content -costs context. It also drifts: as design decisions evolve, the file -either grows (new section per decision) or rots (old sections -contradict the current code). +An outside review (2026-06-04, action item 7) flagged CLAUDE.md as containing +design history that should live in commits and CHANGELOG, not in an +agent-instructions file. Today CLAUDE.md is the single instructions file for +every agent that touches this repo (symlinked from `.cursorrules` etc.), so its +content is read fresh into every agent session — long content costs context. It +also drifts: as design decisions evolve, the file either grows (new section per +decision) or rots (old sections contradict the current code). ## Approach Audit CLAUDE.md and keep only content that's: -1. **A rule the agent must follow** — git workflow, branch - conventions, commit-message format, "never commit autonomously," - working-directory restrictions, etc. -2. **A project-specific fact** the agent can't derive from reading - the code — "this repo's dev loop uses cocogitto," "release uses - cargo-dist," "lance is the storage backend not parquet," - "Refractions is a personal vault, never commit there." -3. **An invariant** the agent must respect — the architectural - commitments that survive across refactors (enum dispatch over - `dyn Trait`, exhaustive matches, decoupled validation layer). +1. **A rule the agent must follow** — git workflow, branch conventions, + commit-message format, "never commit autonomously," working-directory + restrictions, etc. +2. **A project-specific fact** the agent can't derive from reading the code — + "this repo's dev loop uses cocogitto," "release uses cargo-dist," "lance is + the storage backend not parquet," "Refractions is a personal vault, never + commit there." +3. **An invariant** the agent must respect — the architectural commitments that + survive across refactors (enum dispatch over `dyn Trait`, exhaustive matches, + decoupled validation layer). Remove or relocate anything that's: -- **Design history** of waves / TODOs / decisions — belongs in - commits, CHANGELOG, and the assessments / archive directories. -- **Module-by-module descriptions** — derivable from the code via - `cargo doc` or a quick `rg`. Don't recapitulate the codebase. +- **Design history** of waves / TODOs / decisions — belongs in commits, + CHANGELOG, and the assessments / archive directories. +- **Module-by-module descriptions** — derivable from the code via `cargo doc` or + a quick `rg`. Don't recapitulate the codebase. - **Per-feature explanations** of how things work — those belong in - `docs/spec/architecture.md` or per-command spec pages, not in the - agent rules. + `docs/spec/architecture.md` or per-command spec pages, not in the agent rules. -After the trim, CLAUDE.md should be readable end-to-end in well -under a minute. The reviewer's "one screen" framing is the right -direction even if literally one terminal screen is too tight. +After the trim, CLAUDE.md should be readable end-to-end in well under a minute. +The reviewer's "one screen" framing is the right direction even if literally one +terminal screen is too tight. ## Verification -- CLAUDE.md fits in roughly one screenful (~50–80 lines) of prose - plus the conventional-commit reference. -- Every line that survives is either a rule, a project fact, or an - architectural invariant. -- Design history moved to commits / CHANGELOG / archive lives in a - durable place, not lost. -- An agent picking up the repo cold can still infer the rules - needed to behave correctly (don't commit without explicit - authorization, branch before editing, etc.). +- CLAUDE.md fits in roughly one screenful (~50–80 lines) of prose plus the + conventional-commit reference. +- Every line that survives is either a rule, a project fact, or an architectural + invariant. +- Design history moved to commits / CHANGELOG / archive lives in a durable + place, not lost. +- An agent picking up the repo cold can still infer the rules needed to behave + correctly (don't commit without explicit authorization, branch before editing, + etc.). ## Out of scope - The `skills/` directory — each skill stays self-contained. -- The auto-memory in `~/.claude/projects/.../memory/` — that's a - separate persistent surface managed by the agent. +- The auto-memory in `~/.claude/projects/.../memory/` — that's a separate + persistent surface managed by the agent. - AGENTS.md (the symlinked copy) — same file, no separate work. ## Impact -Each agent session reads CLAUDE.md from scratch. A focused file -means the agent's working context starts cleaner, and a reader who -opens the file as part of evaluating the project doesn't conclude -"this is more about how Claude built it than what it does." +Each agent session reads CLAUDE.md from scratch. A focused file means the +agent's working context starts cleaner, and a reader who opens the file as part +of evaluating the project doesn't conclude "this is more about how Claude built +it than what it does." diff --git a/docs/spec/todos/TODO-0184.md b/docs/spec/todos/TODO-0184.md index fe5dfab..5d80d0c 100644 --- a/docs/spec/todos/TODO-0184.md +++ b/docs/spec/todos/TODO-0184.md @@ -26,102 +26,96 @@ files_updated: ## Resolution -Shipped on `feat/mock-embedder`. The design landed mostly as planned with -one refinement: instead of pure `#[cfg(feature = "testing-mocks")]` -gating, the mock is gated with `#[cfg(any(test, feature = "testing-mocks"))]`. -This keeps the production binary mock-free (`cargo build` and -`cargo install` both have `cfg(test) = false` and the feature off → -mock variant doesn't exist) while letting plain `cargo test` work for -local dev without `--features testing-mocks`. CI still passes the feature -explicitly for symmetry with `cargo clippy --all-targets`. Activation in -runtime additionally requires `[embedding_model].provider = "mock"` in -`mdvs.toml`. +Shipped on `feat/mock-embedder`. The design landed mostly as planned with one +refinement: instead of pure `#[cfg(feature = "testing-mocks")]` gating, the mock +is gated with `#[cfg(any(test, feature = "testing-mocks"))]`. This keeps the +production binary mock-free (`cargo build` and `cargo install` both have +`cfg(test) = false` and the feature off → mock variant doesn't exist) while +letting plain `cargo test` work for local dev without +`--features testing-mocks`. CI still passes the feature explicitly for symmetry +with `cargo clippy --all-targets`. Activation in runtime additionally requires +`[embedding_model].provider = "mock"` in `mdvs.toml`. Key pieces: - New `[features] testing-mocks = []` in `crates/mdvs/Cargo.toml`. -- New `MockEmbedder` + `ModelConfig::Mock { dim }` + `Embedder::Mock` - variants in `crates/mdvs/src/index/embed.rs`. Vectors derived from +- New `MockEmbedder` + `ModelConfig::Mock { dim }` + `Embedder::Mock` variants + in `crates/mdvs/src/index/embed.rs`. Vectors derived from `xxh3_64_with_seed(text, counter)` over a counter, reinterpreted to `[-0.5, 0.5]` f32s. No normalization (LanceDB handles cosine norm). - `large_enum_variant` allowed on `Embedder` with a comment — boxing - `Model2Vec` would add a heap allocation to every embed call on the - prod path. + `large_enum_variant` allowed on `Embedder` with a comment — boxing `Model2Vec` + would add a heap allocation to every embed call on the prod path. - `EmbeddingModelConfig.dim: Option` added to `schema/shared.rs`; consulted only by the `mock` provider (default 256). -- `BuildMetadata::to_hash_map` / `from_hash_map` round-trip `dim` as - `mdvs.dim` when present — preserves mock-built indices' shape and - leaves real-model metadata unchanged. -- `mutate_config` in `crates/mdvs/src/cmd/build.rs` writes a mock - default under `cfg(any(test, feature = "testing-mocks"))`, the - `model2vec` default otherwise. This is why `cargo test` works without - threading mock plumbing through every init+build test site. -- Real-model tests in `index/embed.rs` all marked `#[ignore]` with a - uniform message pointing at `cargo test -- --ignored`. New - `mock_tests` module covers the mock surface (dimension, determinism, - distinctness, batch consistency, try_from dispatch). -- `init_and_build` helpers in `cmd/info.rs` and `cmd/search.rs` swap to - mock between init and build — explicit cargo-cult-resistant for the - one or two tests that read back the model name. -- `.github/workflows/ci.yml` runs `cargo test --features testing-mocks` - and `cargo clippy --all-targets --features testing-mocks -- -D warnings`. -- `AGENTS.md` (symlinked from `CLAUDE.md`) Build & Verify section - documents the new commands and the `testing-mocks` feature. +- `BuildMetadata::to_hash_map` / `from_hash_map` round-trip `dim` as `mdvs.dim` + when present — preserves mock-built indices' shape and leaves real-model + metadata unchanged. +- `mutate_config` in `crates/mdvs/src/cmd/build.rs` writes a mock default under + `cfg(any(test, feature = "testing-mocks"))`, the `model2vec` default + otherwise. This is why `cargo test` works without threading mock plumbing + through every init+build test site. +- Real-model tests in `index/embed.rs` all marked `#[ignore]` with a uniform + message pointing at `cargo test -- --ignored`. New `mock_tests` module covers + the mock surface (dimension, determinism, distinctness, batch consistency, + try_from dispatch). +- `init_and_build` helpers in `cmd/info.rs` and `cmd/search.rs` swap to mock + between init and build — explicit cargo-cult-resistant for the one or two + tests that read back the model name. +- `.github/workflows/ci.yml` runs `cargo test --features testing-mocks` and + `cargo clippy --all-targets --features testing-mocks -- -D warnings`. +- `AGENTS.md` (symlinked from `CLAUDE.md`) Build & Verify section documents the + new commands and the `testing-mocks` feature. Verification: - `cargo build` — production, no mock variant in the binary. - `cargo test` — 804 fast-lane tests pass; 12 real-model ignored. - `cargo test --features testing-mocks` — same result. -- `HF_HUB_OFFLINE=1 cargo test --features testing-mocks` — same result. - Fast lane is fully hermetic. -- `cargo test --features testing-mocks -- --ignored` — slow lane runs - locally against the cached HF model. +- `HF_HUB_OFFLINE=1 cargo test --features testing-mocks` — same result. Fast + lane is fully hermetic. +- `cargo test --features testing-mocks -- --ignored` — slow lane runs locally + against the cached HF model. - `cargo clippy --all-targets --features testing-mocks -- -D warnings` — clean. - `cargo fmt --check` — clean. -- Test runtime collapsed from ~3 s to ~0.5 s (no model load in the - fast lane). +- Test runtime collapsed from ~3 s to ~0.5 s (no model load in the fast lane). -One ergonomics call deferred to a separate TODO if it ever becomes -needed: an `actions/cache` step for the HF model directory so a nightly -slow-lane CI job could run. Today it isn't worth the complexity. +One ergonomics call deferred to a separate TODO if it ever becomes needed: an +`actions/cache` step for the HF model directory so a nightly slow-lane CI job +could run. Today it isn't worth the complexity. ## Problem -The current test suite loads the real `minishlab/potion-base-8M` -model from Hugging Face inside CI. Two issues: - -1. **CI hermeticism.** ~30+ tests under `cmd::build::*`, - `cmd::search::*`, and `index::embed::*` resolve the model from HF - on every fresh runner. Anonymous GitHub Actions IP pools hit - HF's rate-limit ceiling intermittently — observed in PR #51 as - ~50 tests failing with `status code 429` on - `huggingface.co/.../tokenizer.json`. The PR had zero code - changes; the fail was pure external flake. As more projects use - HF from GH Actions, the frequency only goes up. - -2. **Test coupling.** Most failing tests aren't testing the - embedder. They're testing `build_core`'s classify / write / - skip paths, or search's query plumbing, and incidentally trigger - a real model load because `build_core` calls `Embedder::load(...)`. - They depend on a network service for behavior that has nothing - to do with cosine values. +The current test suite loads the real `minishlab/potion-base-8M` model from +Hugging Face inside CI. Two issues: + +1. **CI hermeticism.** ~30+ tests under `cmd::build::*`, `cmd::search::*`, and + `index::embed::*` resolve the model from HF on every fresh runner. Anonymous + GitHub Actions IP pools hit HF's rate-limit ceiling intermittently — observed + in PR #51 as ~50 tests failing with `status code 429` on + `huggingface.co/.../tokenizer.json`. The PR had zero code changes; the fail + was pure external flake. As more projects use HF from GH Actions, the + frequency only goes up. + +2. **Test coupling.** Most failing tests aren't testing the embedder. They're + testing `build_core`'s classify / write / skip paths, or search's query + plumbing, and incidentally trigger a real model load because `build_core` + calls `Embedder::load(...)`. They depend on a network service for behavior + that has nothing to do with cosine values. ## Goal Two-tier testing: -- **Fast lane (every commit, runs in CI):** a `MockEmbedder` that - produces deterministic dummy vectors. No network. The ~30+ tests - that incidentally load the model today switch to the mock. -- **Slow lane (local-only, optionally nightly):** a small - integration suite that loads the real model. Stays in the repo, - but is gated so CI doesn't trigger it. +- **Fast lane (every commit, runs in CI):** a `MockEmbedder` that produces + deterministic dummy vectors. No network. The ~30+ tests that incidentally load + the model today switch to the mock. +- **Slow lane (local-only, optionally nightly):** a small integration suite that + loads the real model. Stays in the repo, but is gated so CI doesn't trigger + it. -The fast lane removes the HF dependency from PR-blocking CI -entirely. The slow lane keeps real-model coverage for the -handful of places where it actually matters (dimension mismatch -detection, cosine ranking smoke tests, etc.). +The fast lane removes the HF dependency from PR-blocking CI entirely. The slow +lane keeps real-model coverage for the handful of places where it actually +matters (dimension mismatch detection, cosine ranking smoke tests, etc.). ## Design decisions @@ -129,74 +123,75 @@ Two trade-offs settled before implementation: ### 1. Activation: Cargo feature flag, not always-on config marker -The mock is gated behind a Cargo feature `testing-mocks`, **off by -default**. The variant doesn't exist in production builds. +The mock is gated behind a Cargo feature `testing-mocks`, **off by default**. +The variant doesn't exist in production builds. -Rationale: an always-on `provider = "mock"` config marker would -let a real user (deliberately or by mistake) build an index with -garbage embeddings — a cosmetic foot-gun but not a real one. -Gating behind a feature flag closes the door entirely: -`cargo install mdvs` cannot select the mock because the variant -isn't compiled in. The cost is ~3 lines of Cargo + workflow -plumbing. +Rationale: an always-on `provider = "mock"` config marker would let a real user +(deliberately or by mistake) build an index with garbage embeddings — a cosmetic +foot-gun but not a real one. Gating behind a feature flag closes the door +entirely: `cargo install mdvs` cannot select the mock because the variant isn't +compiled in. The cost is ~3 lines of Cargo + workflow plumbing. Activation requires **both** the feature flag at compile time AND -`[embedding_model].provider = "mock"` in `mdvs.toml` at runtime — -defense in depth. +`[embedding_model].provider = "mock"` in `mdvs.toml` at runtime — defense in +depth. ### 2. Slow-lane gating: `#[ignore]`, not a separate feature -Real-model tests use `#[ignore]`. `cargo test --features testing- -mocks` runs the fast lane only; `cargo test --features testing- -mocks -- --ignored` runs the slow lane locally. +Real-model tests use `#[ignore]`. `cargo test --features testing- mocks` runs +the fast lane only; `cargo test --features testing- mocks -- --ignored` runs the +slow lane locally. -Rationale: idiomatic Rust, no extra plumbing, no nightly CI -machinery needed for v1. +Rationale: idiomatic Rust, no extra plumbing, no nightly CI machinery needed for +v1. ## Approach ### Step 1 — Cargo feature + Mock types `crates/mdvs/Cargo.toml`: + ```toml [features] testing-mocks = [] ``` `crates/mdvs/src/index/embed.rs`: + - Add `#[cfg(feature = "testing-mocks")] ModelConfig::Mock { dim: usize }`. - Add `#[cfg(feature = "testing-mocks")] Embedder::Mock(MockEmbedder)`. -- Implement `MockEmbedder { dim: usize }` with deterministic - vectors derived from `xxh3_64(text)` seeding a `[u8; dim*4]` - buffer reinterpreted as `Vec`. Same input → same vector; - distinct inputs → distinct vectors. No normalization (LanceDB - handles cosine norm). -- Extend all four matches in `Embedder` (`load`, `dimension`, - `embed`, `embed_batch`) with feature-gated arms. The `dimension` - branch for the mock returns its configured `dim` directly — must - not call `encode_single("probe")`-style invocations. +- Implement `MockEmbedder { dim: usize }` with deterministic vectors derived + from `xxh3_64(text)` seeding a `[u8; dim*4]` buffer reinterpreted as + `Vec`. Same input → same vector; distinct inputs → distinct vectors. No + normalization (LanceDB handles cosine norm). +- Extend all four matches in `Embedder` (`load`, `dimension`, `embed`, + `embed_batch`) with feature-gated arms. The `dimension` branch for the mock + returns its configured `dim` directly — must not call + `encode_single("probe")`-style invocations. ### Step 2 — Config plumbing -`crates/mdvs/src/schema/shared.rs` — add an optional `dim: Option` -to `EmbeddingModelConfig` (only consulted by the mock branch; -ignored for `model2vec`; doesn't break existing configs). +`crates/mdvs/src/schema/shared.rs` — add an optional `dim: Option` to +`EmbeddingModelConfig` (only consulted by the mock branch; ignored for +`model2vec`; doesn't break existing configs). `crates/mdvs/src/index/embed.rs` — extend `impl TryFrom<&EmbeddingModelConfig> for ModelConfig`: -- Under `#[cfg(feature = "testing-mocks")]`, match - `provider == "mock"` → `ModelConfig::Mock { dim: config.dim.unwrap_or(256) }`. + +- Under `#[cfg(feature = "testing-mocks")]`, match `provider == "mock"` → + `ModelConfig::Mock { dim: config.dim.unwrap_or(256) }`. - Without the feature, `"mock"` falls through to the existing `unsupported embedding provider` bail, with a hint: "build with `--features testing-mocks` to enable the mock embedder for tests". -`mdvs info` (`crates/mdvs/src/cmd/info.rs`) renders `provider` -honestly — a vault built with the mock prints `provider: mock` in -its info output. Don't hide it. +`mdvs info` (`crates/mdvs/src/cmd/info.rs`) renders `provider` honestly — a +vault built with the mock prints `provider: mock` in its info output. Don't hide +it. ### Step 3 — CI workflow `.github/workflows/ci.yml`: + - Change `cargo test` → `cargo test --features testing-mocks`. - Change `cargo clippy --all-targets -- -D warnings` → `cargo clippy --all-targets --features testing-mocks -- -D warnings`. @@ -206,11 +201,10 @@ its info output. Don't hide it. ### Step 4 — Dev docs - `CLAUDE.md` "Build & Verify" section: replace `cargo test` and - `cargo clippy --all-targets` with the feature-flagged versions. - Add a one-line note about the slow lane: - `cargo test --features testing-mocks -- --ignored`. -- `.claude/skills/code-editing/SKILL.md`: same update if it - references the test command. + `cargo clippy --all-targets` with the feature-flagged versions. Add a one-line + note about the slow lane: `cargo test --features testing-mocks -- --ignored`. +- `.claude/skills/code-editing/SKILL.md`: same update if it references the test + command. ### Step 5 — Convert existing tests @@ -220,88 +214,83 @@ Inventory (from `rg 'Embedder::load|EmbeddingModelConfig {'` in For each, classify: -- **Embedding-behavior** (cosine values, model identity, real - dimension assertion) → add `#[ignore]`. Slow lane. - - All tests in `index/embed.rs` lines 124–236 (load, dimension, - deterministic, similar-texts-higher-cosine, etc.). +- **Embedding-behavior** (cosine values, model identity, real dimension + assertion) → add `#[ignore]`. Slow lane. + - All tests in `index/embed.rs` lines 124–236 (load, dimension, deterministic, + similar-texts-higher-cosine, etc.). - `cmd/search.rs:566, 594` (cosine ranking). - - Any test asserting on dimension number, vector content, or - model-mismatch detection in `cmd/build.rs`. -- **Pipeline-behavior** (classify, write_index decisions, search - result count, error handling, `auto_*` chains) → switch - `EmbeddingModelConfig` to `provider: "mock", dim: Some(256)`. + - Any test asserting on dimension number, vector content, or model-mismatch + detection in `cmd/build.rs`. +- **Pipeline-behavior** (classify, `write_index` decisions, search result count, + error handling, `auto_*` chains) → switch `EmbeddingModelConfig` to + `provider: "mock", dim: Some(256)`. - Most of `cmd/build.rs:1567, 1649, 1712`. - `cmd/info.rs:316`. - `cmd/search.rs:397`. -Direct callers of `Embedder::load(&ModelConfig::Model2Vec { ... })` -literals always go to the slow lane. +Direct callers of `Embedder::load(&ModelConfig::Model2Vec { ... })` literals +always go to the slow lane. -A small test-helper added near the existing config-builder helpers -in `cmd/build.rs` and `cmd/search.rs` to construct -`EmbeddingModelConfig { provider: "mock".into(), name: "mock".into(), -revision: None, dim: Some(256) }` keeps each call site one line. +A small test-helper added near the existing config-builder helpers in +`cmd/build.rs` and `cmd/search.rs` to construct +`EmbeddingModelConfig { provider: "mock".into(), name: "mock".into(), revision: None, dim: Some(256) }` +keeps each call site one line. ### Step 6 — Verify -- `HF_HUB_OFFLINE=1 cargo test --features testing-mocks` passes - with zero network access. Fast lane fully hermetic. -- `cargo test --features testing-mocks -- --ignored` runs the - slow lane locally; real-model tests pass with cached weights. -- `cargo install --path crates/mdvs` (no feature flags) builds - successfully without the mock variant in the binary. +- `HF_HUB_OFFLINE=1 cargo test --features testing-mocks` passes with zero + network access. Fast lane fully hermetic. +- `cargo test --features testing-mocks -- --ignored` runs the slow lane locally; + real-model tests pass with cached weights. +- `cargo install --path crates/mdvs` (no feature flags) builds successfully + without the mock variant in the binary. - Push the branch; CI runs green without HF in the picture. ## Risks watched during implementation -- **Dimension shortcut.** The real-model `Embedder::dimension()` - calls `encode_single("probe").len()`. The mock branch must - return its configured `dim` directly — never invoke any - encoding path. -- **Build metadata mismatch.** Build metadata writes - `EmbeddingModelConfig` into the Lance dataset. A mock-built - index opened by a real-provider config triggers the model- - identity check. Tests using the mock must be self-contained - (build + query under the same mock config); already true for +- **Dimension shortcut.** The real-model `Embedder::dimension()` calls + `encode_single("probe").len()`. The mock branch must return its configured + `dim` directly — never invoke any encoding path. +- **Build metadata mismatch.** Build metadata writes `EmbeddingModelConfig` into + the Lance dataset. A mock-built index opened by a real-provider config + triggers the model- identity check. Tests using the mock must be + self-contained (build + query under the same mock config); already true for the tests being migrated. -- **Cosine-value assertions.** Any test asserting on cosine - *values* (not just ordering) is automatically slow-lane — the - mock's vectors are noise. The classification above catches - these. +- **Cosine-value assertions.** Any test asserting on cosine _values_ (not just + ordering) is automatically slow-lane — the mock's vectors are noise. The + classification above catches these. ## Verification (final) -- `cargo test --features testing-mocks` (default, no ignored) - passes without network access. Test in airplane mode: `mdvs check` - and the full test suite both succeed. -- `cargo test --features testing-mocks -- --ignored` runs the slow - lane locally and the real-model tests pass with cached weights. -- The 30+ tests that today reach HF via `Embedder::load(...)` now - use the mock and make zero network calls. Verified by running - them under `HF_HUB_OFFLINE=1` or with an unreachable HF URL. +- `cargo test --features testing-mocks` (default, no ignored) passes without + network access. Test in airplane mode: `mdvs check` and the full test suite + both succeed. +- `cargo test --features testing-mocks -- --ignored` runs the slow lane locally + and the real-model tests pass with cached weights. +- The 30+ tests that today reach HF via `Embedder::load(...)` now use the mock + and make zero network calls. Verified by running them under `HF_HUB_OFFLINE=1` + or with an unreachable HF URL. - CI on this same PR runs green without HF flakes. - `cargo install` of the production binary doesn't ship the mock. ## Out of scope -- Building a CI cache for the HF model directory — irrelevant - once the fast lane is hermetic, and unnecessary if the slow - lane only runs locally. -- Adding an `HF_TOKEN` secret — same reason; only needed if the - slow lane ever moves into CI. +- Building a CI cache for the HF model directory — irrelevant once the fast lane + is hermetic, and unnecessary if the slow lane only runs locally. +- Adding an `HF_TOKEN` secret — same reason; only needed if the slow lane ever + moves into CI. - Refactoring `Embedder` beyond adding the feature-gated variant. -- Mocking other external dependencies (LanceDB writes to a - tempdir; that's already hermetic). -- A nightly online-tests CI workflow — can be added later if the - slow lane ever catches something the fast lane missed. +- Mocking other external dependencies (LanceDB writes to a tempdir; that's + already hermetic). +- A nightly online-tests CI workflow — can be added later if the slow lane ever + catches something the fast lane missed. ## Impact -This is the structural fix for the HF rate-limit flake that hit -PR #51. Combined with no other change, it removes the external -dependency from PR-blocking CI, makes the fast lane meaningfully -faster, and keeps the real-model coverage intact for local / -nightly runs. The feature-flag gating also closes the cosmetic -foot-gun where a user could (deliberately or by mistake) build a -real index with mock vectors. Probably the highest-leverage CI -improvement we can make before public amplification. +This is the structural fix for the HF rate-limit flake that hit PR #51. Combined +with no other change, it removes the external dependency from PR-blocking CI, +makes the fast lane meaningfully faster, and keeps the real-model coverage +intact for local / nightly runs. The feature-flag gating also closes the +cosmetic foot-gun where a user could (deliberately or by mistake) build a real +index with mock vectors. Probably the highest-leverage CI improvement we can +make before public amplification. diff --git a/docs/spec/todos/TODO-0185.md b/docs/spec/todos/TODO-0185.md index 257b3bf..245cd20 100644 --- a/docs/spec/todos/TODO-0185.md +++ b/docs/spec/todos/TODO-0185.md @@ -15,93 +15,85 @@ files_updated: ## Resolution -Surgical edits to `crates/mdvs/skills/mdvs/SKILL.md` covering the -nine drift points listed below. No code changed; `cargo test` and +Surgical edits to `crates/mdvs/skills/mdvs/SKILL.md` covering the nine drift +points listed below. No code changed; `cargo test` and `cargo clippy --all-targets` stayed green. Verified by running -`mdvs skill | diff - crates/mdvs/skills/mdvs/SKILL.md` — bytes -match, so the in-binary copy is also current. +`mdvs skill | diff - crates/mdvs/skills/mdvs/SKILL.md` — bytes match, so the +in-binary copy is also current. ## Problem -`crates/mdvs/skills/mdvs/SKILL.md` is the canonical agent-facing -description of mdvs — it's the file other agents read when they -load mdvs as a skill (and what `mdvs skill` prints to stdout). It -has drifted off the v0.7.0 surface in several places: +`crates/mdvs/skills/mdvs/SKILL.md` is the canonical agent-facing description of +mdvs — it's the file other agents read when they load mdvs as a skill (and what +`mdvs skill` prints to stdout). It has drifted off the v0.7.0 surface in several +places: -- **Missing command** — `mdvs export-jsonschema [path] [--format - json|toml] [--output-file FILE]` is not in the command table or - the reference section. Shipped in Wave B. -- **Missing command** — `mdvs skill` (prints the skill file to - stdout) is not listed either. -- **Missing flags** — `init --from-jsonschema PATH` (import an - external JSON Schema instead of inferring) and `check - --jsonschema PATH` (override `[fields]` for one run) are not - mentioned. -- **Stale field-type list** — line ~197 lists `String, Integer, - Float, Boolean, Array(T), Object(...)`. Missing **`Date`** and - **`DateTime`** (shipped TODO-0007, 2026-05-14). +- **Missing command** — + `mdvs export-jsonschema [path] [--format json|toml] [--output-file FILE]` is + not in the command table or the reference section. Shipped in Wave B. +- **Missing command** — `mdvs skill` (prints the skill file to stdout) is not + listed either. +- **Missing flags** — `init --from-jsonschema PATH` (import an external JSON + Schema instead of inferring) and `check --jsonschema PATH` (override + `[fields]` for one run) are not mentioned. +- **Stale field-type list** — line ~197 lists + `String, Integer, Float, Boolean, Array(T), Object(...)`. Missing **`Date`** + and **`DateTime`** (shipped TODO-0007, 2026-05-14). - **Wrong Object syntax** — `Object(...)` should be `Object{k: v}` - (function-style with curly braces). Top-level `Object` is also - rejected at config load (Wave C invariant 6); the file says - nothing about **dotted-name leaves** - (`calibration.baseline.wavelength`), which is how nested - frontmatter is actually expressed in `mdvs.toml`. -- **No mention of `preprocess`** — Stage 2 preprocessors - (`coerce_to_string`, `widen_int_to_float`) are now a first-class - per-field property in `mdvs.toml`. Inference auto-populates them - when widening was observed. The file's "mixed types widen" - bullet describes the symptom without explaining that strict - mode rejects unless `preprocess` opts in. -- **Model size off** — "first build downloads the embedding model - (~30 MB)". Current default is `minishlab/potion-base-8M`, closer - to ~60 MB. -- **Constraints listing incomplete** — text mentions categorical - and range only; `length` (min_length / max_length) and `pattern` - also exist. + (function-style with curly braces). Top-level `Object` is also rejected at + config load (Wave C invariant 6); the file says nothing about **dotted-name + leaves** (`calibration.baseline.wavelength`), which is how nested frontmatter + is actually expressed in `mdvs.toml`. +- **No mention of `preprocess`** — Stage 2 preprocessors (`coerce_to_string`, + `widen_int_to_float`) are now a first-class per-field property in `mdvs.toml`. + Inference auto-populates them when widening was observed. The file's "mixed + types widen" bullet describes the symptom without explaining that strict mode + rejects unless `preprocess` opts in. +- **Model size off** — "first build downloads the embedding model (~30 MB)". + Current default is `minishlab/potion-base-8M`, closer to ~60 MB. +- **Constraints listing incomplete** — text mentions categorical and range only; + `length` (min_length / max_length) and `pattern` also exist. ## Approach -Surgical edits to bring the file up to date — not a rewrite. Keep -the existing structure (command table → two-layers → command -reference → workflows → things-to-know → examples → common -errors). Insert / amend at the right spots. +Surgical edits to bring the file up to date — not a rewrite. Keep the existing +structure (command table → two-layers → command reference → workflows → +things-to-know → examples → common errors). Insert / amend at the right spots. Concrete edits: -1. **Command table** (the `| User intent | Command |` block): add - rows for `export-jsonschema` and `skill`. -2. **Command reference**: add a short `### mdvs export-jsonschema` - section (path, `--format`, `--output-file`, round-trip with - `init --from-jsonschema`) and a brief `### mdvs skill` note. +1. **Command table** (the `| User intent | Command |` block): add rows for + `export-jsonschema` and `skill`. +2. **Command reference**: add a short `### mdvs export-jsonschema` section + (path, `--format`, `--output-file`, round-trip with `init --from-jsonschema`) + and a brief `### mdvs skill` note. 3. **`mdvs init`** subsection: document `--from-jsonschema PATH`. 4. **`mdvs check`** subsection: document `--jsonschema PATH`. 5. **Things to know**: rewrite the field-types bullet to list - `String, Integer, Float, Boolean, Date, DateTime, Array(T), - Array(Object{...})`, note that top-level Object is rejected, - and explain dotted-name leaves with one example. -6. **Things to know**: add a `preprocess` bullet — what the two - built-in stages are, that inference auto-populates them, and - that strict mode rejects widening without an opt-in. -7. **Things to know**: extend the constraints bullet to include - `length` and `pattern`. + `String, Integer, Float, Boolean, Date, DateTime, Array(T), Array(Object{...})`, + note that top-level Object is rejected, and explain dotted-name leaves with + one example. +6. **Things to know**: add a `preprocess` bullet — what the two built-in stages + are, that inference auto-populates them, and that strict mode rejects + widening without an opt-in. +7. **Things to know**: extend the constraints bullet to include `length` and + `pattern`. 8. **`mdvs build`** subsection: fix the model size to ~60 MB. ## Verification -- `cargo run -- skill` output matches the edited file byte-for- - byte (the `skill` subcommand should print exactly this file). -- Grep the edited file for `parquet` → no hits (already clean, - spot-check on edit). +- `cargo run -- skill` output matches the edited file byte-for- byte (the + `skill` subcommand should print exactly this file). +- Grep the edited file for `parquet` → no hits (already clean, spot-check on + edit). - `cargo test` still green; no behavior change, no code touched. -- Read end-to-end as another agent would on cold load — every - flag, command, and field type mentioned in `--help` appears - somewhere in the skill. +- Read end-to-end as another agent would on cold load — every flag, command, and + field type mentioned in `--help` appears somewhere in the skill. ## Out of scope -- The other in-repo skills under `.claude/skills/` (book, - code-editing, commit, etc.) — those are agent-workflow skills, - not user-facing tool surface. -- Restructure or trim the file. v0 of the rewrite stays - surgically minimal; a fuller pass can happen later if needed. +- The other in-repo skills under `.claude/skills/` (book, code-editing, commit, + etc.) — those are agent-workflow skills, not user-facing tool surface. +- Restructure or trim the file. v0 of the rewrite stays surgically minimal; a + fuller pass can happen later if needed. - `book/` or `docs/spec/` drift — TODO-0176 covers that. diff --git a/docs/spec/todos/TODO-0186.md b/docs/spec/todos/TODO-0186.md index aad2779..e77e359 100644 --- a/docs/spec/todos/TODO-0186.md +++ b/docs/spec/todos/TODO-0186.md @@ -15,23 +15,21 @@ related: [190] Today an agent that wants to write a new note has two options: -1. **Guess + retry.** Compose frontmatter, run `mdvs check`, parse - violations, fix, repeat. Reactive. -2. **Read `mdvs.toml` and reverse-engineer scope.** Parse the TOML, - evaluate every field's `allowed` / `required` globs against the - target path, derive the applicable rules. Doable but it requires - the agent (or the harness writing the agent prompt) to reimplement - mdvs's path-scoping semantics. Brittle. - -Neither is what an agent harness actually wants. The shape that -matches the use case is: "I'm about to write `notes/projects/alpha/ -draft.md` — what fields are allowed / required at that path, and -what constraints govern each?" One call, structured answer, agent -generates valid frontmatter on the first try. - -This pairs with TODO-0177's audience reframe (LLM-curated KBs) and -makes the agent-callable surface concrete. It's the proactive twin -of `mdvs check`. +1. **Guess + retry.** Compose frontmatter, run `mdvs check`, parse violations, + fix, repeat. Reactive. +2. **Read `mdvs.toml` and reverse-engineer scope.** Parse the TOML, evaluate + every field's `allowed` / `required` globs against the target path, derive + the applicable rules. Doable but it requires the agent (or the harness + writing the agent prompt) to reimplement mdvs's path-scoping semantics. + Brittle. + +Neither is what an agent harness actually wants. The shape that matches the use +case is: "I'm about to write `notes/projects/alpha/ draft.md` — what fields are +allowed / required at that path, and what constraints govern each?" One call, +structured answer, agent generates valid frontmatter on the first try. + +This pairs with TODO-0177's audience reframe (LLM-curated KBs) and makes the +agent-callable surface concrete. It's the proactive twin of `mdvs check`. ## Goal @@ -41,23 +39,22 @@ New subcommand: mdvs explain [--output pretty|markdown|json] ``` -`` is the candidate file path (need not exist). Output -reports the fields whose `allowed` glob matches ``, with -their type, nullability, constraints, preprocess pipeline, and -whether they are `required` at that path. The response also reports -fields known to mdvs.toml that are *not* applicable here (so the -agent knows what NOT to write), and a textual `scope_note` for -edge cases. +`` is the candidate file path (need not exist). Output reports the +fields whose `allowed` glob matches ``, with their type, nullability, +constraints, preprocess pipeline, and whether they are `required` at that path. +The response also reports fields known to mdvs.toml that are _not_ applicable +here (so the agent knows what NOT to write), and a textual `scope_note` for edge +cases. -`--output pretty` is human-readable (box-drawing tables); -`--output markdown` is the same content as GFM tables, suitable for -piping or agent consumption; `--output json` is the strict structured -contract for `jq` or programmatic consumption. +`--output pretty` is human-readable (box-drawing tables); `--output markdown` is +the same content as GFM tables, suitable for piping or agent consumption; +`--output json` is the strict structured contract for `jq` or programmatic +consumption. ### `` must be a file path, not a directory -`explain` answers a per-file question. Directories don't map to -globs cleanly. If `` is a directory: +`explain` answers a per-file question. Directories don't map to globs cleanly. +If `` is a directory: ``` error: `mdvs explain` requires a file path. For a directory-tree @@ -65,10 +62,10 @@ overview of which rules apply where, use `mdvs explain --tree ` (see TODO-0186-tree-followup). ``` -A directory-tree overview is a different operation (walk the tree, -group fields by glob bucket, show schema landscape). Capture it as -a follow-up TODO if agent users actually ask for it; the per-file -form is what the proactive-write loop needs. +A directory-tree overview is a different operation (walk the tree, group fields +by glob bucket, show schema landscape). Capture it as a follow-up TODO if agent +users actually ask for it; the per-file form is what the proactive-write loop +needs. ## Proposed JSON shape @@ -114,116 +111,104 @@ form is what the proactive-write loop needs. ### Three categories, by intent -The split is deliberate. Each category answers a different question -the agent is implicitly asking: +The split is deliberate. Each category answers a different question the agent is +implicitly asking: -| Category | What it tells the agent | Cost in JSON | -|---|---|---| -| `applicable_fields` | What you should USE — full schema with type, constraints, required, preprocess. | Full schema, ~50–200 bytes per field. | -| `disallowed_fields` | What you must NOT use here unless you first change `allowed` in mdvs.toml. Names only — no full schema needed; the agent can call `mdvs explain` on the field's typical path if it wants the details. | Just a string list, ~10–20 bytes per name. | -| Implicit: "new fields" | Anything not in either list is a never-before-seen field. mdvs will report it as informational (not a violation); `mdvs update` would later add it to the schema. | Not in JSON at all — the agent infers this from "not present in either list". | +| Category | What it tells the agent | Cost in JSON | +| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| `applicable_fields` | What you should USE — full schema with type, constraints, required, preprocess. | Full schema, ~50–200 bytes per field. | +| `disallowed_fields` | What you must NOT use here unless you first change `allowed` in mdvs.toml. Names only — no full schema needed; the agent can call `mdvs explain` on the field's typical path if it wants the details. | Just a string list, ~10–20 bytes per name. | +| Implicit: "new fields" | Anything not in either list is a never-before-seen field. mdvs will report it as informational (not a violation); `mdvs update` would later add it to the schema. | Not in JSON at all — the agent infers this from "not present in either list". | -`scope_note` carries the textual hint that bridges the three -categories — especially important for the edge case where -`applicable_fields` is empty (e.g., the agent created a new folder -at the vault root and is writing the first note there). The note -makes it explicit: "no rules apply yet; write whatever, then run -`mdvs update` to bring the schema in line — or set `allowed = ["**"]` -on the fields you want to apply globally." +`scope_note` carries the textual hint that bridges the three categories — +especially important for the edge case where `applicable_fields` is empty (e.g., +the agent created a new folder at the vault root and is writing the first note +there). The note makes it explicit: "no rules apply yet; write whatever, then +run `mdvs update` to bring the schema in line — or set `allowed = ["**"]` on the +fields you want to apply globally." ## Approach -1. **Resolve vault.** Walk up from `` looking for - `mdvs.toml`; error cleanly if none found. +1. **Resolve vault.** Walk up from `` looking for `mdvs.toml`; error + cleanly if none found. 2. **Load schema.** Reuse the existing `MdvsToml::read()` path. 3. **Partition fields.** For each `[[fields.field]]`, check whether - `` (relative to the vault root) matches any `allowed` - glob. - - **Match → `applicable_fields`** with full schema. Mark - `required: true` if `` also matches any of its - `required` globs. + `` (relative to the vault root) matches any `allowed` glob. + - **Match → `applicable_fields`** with full schema. Mark `required: true` if + `` also matches any of its `required` globs. - **No match → `disallowed_fields`** (name only). -4. **Compose `scope_note`.** A short textual summary keyed on the - sizes of the two categories (and whether the field is in a "bare - files allowed" path). -5. **Render output.** Text via the existing `Block` / `Render` - infrastructure (probably a small table); JSON via a new - `outcome::commands::ExplainOutcome` struct + serde. +4. **Compose `scope_note`.** A short textual summary keyed on the sizes of the + two categories (and whether the field is in a "bare files allowed" path). +5. **Render output.** Text via the existing `Block` / `Render` infrastructure + (probably a small table); JSON via a new `outcome::commands::ExplainOutcome` + struct + serde. -The path-scoping logic is already implemented in -`schema/constraints/` and used by `cmd::check::validate`. Reuse it -exactly — no parallel re-implementation. +The path-scoping logic is already implemented in `schema/constraints/` and used +by `cmd::check::validate`. Reuse it exactly — no parallel re-implementation. ## Design questions to settle before implementation -1. **Command name** — `explain` chosen. Reads as an agent verb, - short, unambiguous. Alternative `fields ` is more literal - but loses the "tell me about" flavor. Keeping `explain`. -2. **Canonical JSON Schema slice** — should `--canonical` (or - `--json-schema`) flag emit the JSON Schema 2020-12 subset for - just the applicable fields? Useful for an agent feeding the - schema into a structured-output generator. Defer to v2 unless an - early caller needs it. -3. **Example skeleton** — should the JSON include an - `example_skeleton` field (ready-to-paste frontmatter block with - required fields and sensible defaults)? Helpful for "write me a - new note" agent flows but adds template logic. Decision: NO in - v1. Add `--with-skeleton` flag in v1.5 if asked for. -4. **Path existence** — `` is treated as a candidate - path; the file need not exist. (Confirmed: this is the whole - point — pre-write query.) -5. **Empty `applicable_fields` + non-empty `disallowed_fields`** — - the agent might be in a new sub-tree where nothing applies but - the vault has other rules elsewhere. `scope_note` makes the - situation legible. The agent's three actions in this case: +1. **Command name** — `explain` chosen. Reads as an agent verb, short, + unambiguous. Alternative `fields ` is more literal but loses the "tell + me about" flavor. Keeping `explain`. +2. **Canonical JSON Schema slice** — should `--canonical` (or `--json-schema`) + flag emit the JSON Schema 2020-12 subset for just the applicable fields? + Useful for an agent feeding the schema into a structured-output generator. + Defer to v2 unless an early caller needs it. +3. **Example skeleton** — should the JSON include an `example_skeleton` field + (ready-to-paste frontmatter block with required fields and sensible + defaults)? Helpful for "write me a new note" agent flows but adds template + logic. Decision: NO in v1. Add `--with-skeleton` flag in v1.5 if asked for. +4. **Path existence** — `` is treated as a candidate path; the file + need not exist. (Confirmed: this is the whole point — pre-write query.) +5. **Empty `applicable_fields` + non-empty `disallowed_fields`** — the agent + might be in a new sub-tree where nothing applies but the vault has other + rules elsewhere. `scope_note` makes the situation legible. The agent's three + actions in this case: - Write any frontmatter — none of it will fail validation. - - Or pick a field from `disallowed_fields` and first edit - `mdvs.toml` to extend its `allowed` glob to cover this path. + - Or pick a field from `disallowed_fields` and first edit `mdvs.toml` to + extend its `allowed` glob to cover this path. - Or skip frontmatter entirely (bare file). ## Verification -- `mdvs explain` on a path inside `example_kb/` returns a sensible - partition — e.g., `example_kb/projects/alpha/notes/foo.md` should - show `author`, `status`, `attendees`, etc. in - `applicable_fields`, with `firmware_version` and +- `mdvs explain` on a path inside `example_kb/` returns a sensible partition — + e.g., `example_kb/projects/alpha/notes/foo.md` should show `author`, `status`, + `attendees`, etc. in `applicable_fields`, with `firmware_version` and `wavelength_nm` in `disallowed_fields`. -- `mdvs explain example_kb/some-new-area/note.md` (a path covered - by NO existing `allowed` globs) returns - `applicable_fields: []` plus the right `scope_note`. -- `mdvs explain example_kb/` (a directory path) errors with the - pointer to the tree-overview follow-up. +- `mdvs explain example_kb/some-new-area/note.md` (a path covered by NO existing + `allowed` globs) returns `applicable_fields: []` plus the right `scope_note`. +- `mdvs explain example_kb/` (a directory path) errors with the pointer to the + tree-overview follow-up. - `--output json` is `jq`-parseable and stable across releases. -- New integration test in `crates/mdvs/tests/` confirms the - filtering matches the same predicate that `cmd::check::validate` - uses (no parallel implementation drift). +- New integration test in `crates/mdvs/tests/` confirms the filtering matches + the same predicate that `cmd::check::validate` uses (no parallel + implementation drift). - `cargo test --features testing-mocks` + clippy + fmt clean. - Help text on the new subcommand is concise; example in book. ## Out of scope -- Live validation against proposed frontmatter content (i.e., - "here's my draft frontmatter, tell me what's wrong"). That's - what [TODO-0188](TODO-0188.md) (`mdvs check --stdin`) covers. -- Multiple paths in one call. `for f in files; do mdvs explain $f - --output json; done` handles batch usage; a `--paths-from-stdin` - flag is a v2. -- Directory-tree overview (`mdvs explain --tree `). Captured - as a follow-up TODO if agent users ask for it. +- Live validation against proposed frontmatter content (i.e., "here's my draft + frontmatter, tell me what's wrong"). That's what [TODO-0188](TODO-0188.md) + (`mdvs check --stdin`) covers. +- Multiple paths in one call. + `for f in files; do mdvs explain $f --output json; done` handles batch usage; + a `--paths-from-stdin` flag is a v2. +- Directory-tree overview (`mdvs explain --tree `). Captured as a follow-up + TODO if agent users ask for it. ## Impact -Closes the gap between "we have a schema" and "an agent can use the -schema." Without this, the agent loop is write→check→fix→repeat; -with it, the agent gets the rules upfront and writes valid -frontmatter on the first try. Direct enabler for [TODO-0187] -(agent harness hook recipe) — especially its pre-explain mode. +Closes the gap between "we have a schema" and "an agent can use the schema." +Without this, the agent loop is write→check→fix→repeat; with it, the agent gets +the rules upfront and writes valid frontmatter on the first try. Direct enabler +for [TODO-0187] (agent harness hook recipe) — especially its pre-explain mode. ### Follow-up: pre-explain hook mode (v2) -[TODO-0190](TODO-0190.md) ships `mdvs scaffold hook` with v1 -(post-check) only. **v2 (pre-explain)** — the `PreToolUse` hook -that injects the applicable schema into agent context before the -edit — needs this TODO (`mdvs explain`) to exist. Once 0186 lands, -open a follow-up TODO to add the v2 mode to the per-platform +[TODO-0190](TODO-0190.md) ships `mdvs scaffold hook` with v1 (post-check) only. +**v2 (pre-explain)** — the `PreToolUse` hook that injects the applicable schema +into agent context before the edit — needs this TODO (`mdvs explain`) to exist. +Once 0186 lands, open a follow-up TODO to add the v2 mode to the per-platform scaffolding hook scripts. diff --git a/docs/spec/todos/TODO-0187.md b/docs/spec/todos/TODO-0187.md index 40e9070..8b9c22d 100644 --- a/docs/spec/todos/TODO-0187.md +++ b/docs/spec/todos/TODO-0187.md @@ -14,80 +14,82 @@ subsumed_by: 190 ## Resolution -Subsumed by [TODO-0190](TODO-0190.md). The `mdvs scaffold hook` command surface and the per-harness recipe pages under `book/src/recipes/agent-harnesses/` ship exactly the auto-check recipe this TODO proposed, plus the cross-platform `mdvs hook handle` runtime (which 0187 sketched as a hand-written shell script). The hook-mode taxonomy (auto-check shipped, auto-explain and pre-validate deferred) is preserved as part of 0190's `--kind` design and the deferral lives in TODO-0186 / TODO-0188. +Subsumed by [TODO-0190](TODO-0190.md). The `mdvs scaffold hook` command surface +and the per-harness recipe pages under `book/src/recipes/agent-harnesses/` ship +exactly the auto-check recipe this TODO proposed, plus the cross-platform +`mdvs hook handle` runtime (which 0187 sketched as a hand-written shell script). +The hook-mode taxonomy (auto-check shipped, auto-explain and pre-validate +deferred) is preserved as part of 0190's `--kind` design and the deferral lives +in TODO-0186 / TODO-0188. ## Original scope ## Problem -Today, an agent that writes markdown into an mdvs vault needs to be -*told* to run `mdvs check`. Either the user prompt has to include -"check after every write", or the agent has to remember on its own. -Both are unreliable; both add friction. +Today, an agent that writes markdown into an mdvs vault needs to be _told_ to +run `mdvs check`. Either the user prompt has to include "check after every +write", or the agent has to remember on its own. Both are unreliable; both add +friction. -The right shape is an automatic hook in the agent harness that -plumbs mdvs into the tool-call lifecycle: +The right shape is an automatic hook in the agent harness that plumbs mdvs into +the tool-call lifecycle: -- **After** the agent writes / edits a `.md` file: run `mdvs check` - on the vault, surface violations as tool-result feedback. Agent - fixes in its next turn. -- **Optionally before**: inject the applicable schema (via - [TODO-0186] `mdvs explain`) so the agent writes valid frontmatter - on the first try. -- **Optionally pre-validate** (depends on [TODO-0188] `mdvs check - --stdin`): validate the proposed file content *before* the write - reaches disk, blocking bad writes entirely. +- **After** the agent writes / edits a `.md` file: run `mdvs check` on the + vault, surface violations as tool-result feedback. Agent fixes in its next + turn. +- **Optionally before**: inject the applicable schema (via [TODO-0186] + `mdvs explain`) so the agent writes valid frontmatter on the first try. +- **Optionally pre-validate** (depends on [TODO-0188] `mdvs check --stdin`): + validate the proposed file content _before_ the write reaches disk, blocking + bad writes entirely. -Target harness: Claude Code (PreToolUse / PostToolUse hooks). Same -pattern is portable to other harnesses with minor adjustments — -documented as a footnote in the recipe. +Target harness: Claude Code (PreToolUse / PostToolUse hooks). Same pattern is +portable to other harnesses with minor adjustments — documented as a footnote in +the recipe. ## Goal -Ship a recipe (book page + ready-to-use scripts bundled in the crate) -that wires mdvs into an agent harness's tool-call lifecycle. The -recipe covers three composable modes and lets users pick what fits -their workflow. +Ship a recipe (book page + ready-to-use scripts bundled in the crate) that wires +mdvs into an agent harness's tool-call lifecycle. The recipe covers three +composable modes and lets users pick what fits their workflow. ## The three modes -| Mode | Hook event | Tool payload it inspects | What it does | Cost | Depends on | -|---|---|---|---|---|---| -| **Post-check** | `PostToolUse` on `Write\|Edit\|MultiEdit` | `file_path` | File is already on disk. Walk up to find `mdvs.toml`. Run `mdvs check` on the vault. If violations exist, exit 2 with JSON on stderr — agent sees them next turn and fixes. | Reactive: one wasted write. ~8 ms validation (post-TODO-0172). | nothing — ships today. | -| **Pre-explain** | `PreToolUse` on `Write\|Edit\|MultiEdit` | `file_path` | Walk up to find `mdvs.toml`. Run `mdvs explain --output json`. Print to stdout (Claude Code injects stdout into agent context). Doesn't block. | Informational. ~600 bytes of JSON per write. Agent writes correctly first time. | [TODO-0186](TODO-0186.md) | -| **Pre-validate** | `PreToolUse` on `Write` (and optionally `Edit` after v3) | `file_path` + `content` | Parse proposed content. Run `mdvs check --stdin ` against it. If invalid, exit 2 with violations on stderr. Agent retries with corrected content. **Blocks bad writes from reaching disk.** | Strongest. ~10 ms. | [TODO-0188](TODO-0188.md) | +| Mode | Hook event | Tool payload it inspects | What it does | Cost | Depends on | +| ---------------- | -------------------------------------------------------- | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------- | ------------------------- | +| **Post-check** | `PostToolUse` on `Write\|Edit\|MultiEdit` | `file_path` | File is already on disk. Walk up to find `mdvs.toml`. Run `mdvs check` on the vault. If violations exist, exit 2 with JSON on stderr — agent sees them next turn and fixes. | Reactive: one wasted write. ~8 ms validation (post-TODO-0172). | nothing — ships today. | +| **Pre-explain** | `PreToolUse` on `Write\|Edit\|MultiEdit` | `file_path` | Walk up to find `mdvs.toml`. Run `mdvs explain --output json`. Print to stdout (Claude Code injects stdout into agent context). Doesn't block. | Informational. ~600 bytes of JSON per write. Agent writes correctly first time. | [TODO-0186](TODO-0186.md) | +| **Pre-validate** | `PreToolUse` on `Write` (and optionally `Edit` after v3) | `file_path` + `content` | Parse proposed content. Run `mdvs check --stdin ` against it. If invalid, exit 2 with violations on stderr. Agent retries with corrected content. **Blocks bad writes from reaching disk.** | Strongest. ~10 ms. | [TODO-0188](TODO-0188.md) | ### Rollout sequence - **v1 (now)**: post-check ships. Works today. -- **v2 (after TODO-0186 merges)**: pre-explain hook ships. Recipe - page documents how to enable both modes alongside each other. -- **v3 (after TODO-0188 merges)**: pre-validate hook ships for - `Write`. Edit-aware pre-validate (patch-applies in memory) is a - v3+ stretch. +- **v2 (after TODO-0186 merges)**: pre-explain hook ships. Recipe page documents + how to enable both modes alongside each other. +- **v3 (after TODO-0188 merges)**: pre-validate hook ships for `Write`. + Edit-aware pre-validate (patch-applies in memory) is a v3+ stretch. ## Approach ### Two repo artifacts -1. **`crates/mdvs/hooks/` directory** — ready-to-use scripts. Added - to the `include = [...]` list in `crates/mdvs/Cargo.toml` so they - ship with the bundle (same pattern as `skills/`). A user who runs - `cargo install mdvs` gets the scripts under their cargo registry - directory; we'll document the canonical path in the recipe. +1. **`crates/mdvs/hooks/` directory** — ready-to-use scripts. Added to the + `include = [...]` list in `crates/mdvs/Cargo.toml` so they ship with the + bundle (same pattern as `skills/`). A user who runs `cargo install mdvs` gets + the scripts under their cargo registry directory; we'll document the + canonical path in the recipe. - `crates/mdvs/hooks/mdvs-check.sh` — post-check (v1) - `crates/mdvs/hooks/mdvs-explain.sh` — pre-explain (v2) - `crates/mdvs/hooks/mdvs-validate.sh` — pre-validate (v3) - - `crates/mdvs/hooks/README.md` — per-script setup notes, target- - harness compatibility table. -2. **`book/src/recipes/agent-harness-hook.md`** — recipe page. The - page explains the three modes, shows Claude Code config snippets, - and links to the bundled scripts. + - `crates/mdvs/hooks/README.md` — per-script setup notes, target- harness + compatibility table. +2. **`book/src/recipes/agent-harness-hook.md`** — recipe page. The page explains + the three modes, shows Claude Code config snippets, and links to the bundled + scripts. -A future `mdvs hooks` subcommand (parallel to `mdvs skill`) could -print the bundled scripts' path or `--install` them into a target -harness config. Defer to v2 as a follow-up — v1 keeps the manual -symlink / copy path. +A future `mdvs hooks` subcommand (parallel to `mdvs skill`) could print the +bundled scripts' path or `--install` them into a target harness config. Defer to +v2 as a follow-up — v1 keeps the manual symlink / copy path. ### The post-check script (sketch, ~15 lines) @@ -133,9 +135,9 @@ exit 0 } ``` -`Write|Edit|MultiEdit` is the matcher — covers all three Claude -Code write tools. Edits naturally land in PostToolUse because the -file on disk reflects the post-edit state. +`Write|Edit|MultiEdit` is the matcher — covers all three Claude Code write +tools. Edits naturally land in PostToolUse because the file on disk reflects the +post-edit state. ### Pre-explain sketch (v2) @@ -179,80 +181,72 @@ fi ## Design questions to settle before implementation -1. **Single harness or generic?** Concrete Claude Code config in - the recipe (90% of the audience), short footnote on adapting to - other harnesses. Confirmed in discussion. -2. **Where does the hook script live?** `crates/mdvs/hooks/` in - the repo, included in the bundle (parallel to `skills/`). - Confirmed. +1. **Single harness or generic?** Concrete Claude Code config in the recipe (90% + of the audience), short footnote on adapting to other harnesses. Confirmed in + discussion. +2. **Where does the hook script live?** `crates/mdvs/hooks/` in the repo, + included in the bundle (parallel to `skills/`). Confirmed. 3. **Vault discovery for symlinked / shared workspaces.** The - walk-up-to-`mdvs.toml` strategy handles it per-file. For nested - vaults (a `mdvs.toml` inside another `mdvs.toml`'s tree), the - inner one wins. Document the precedence; nested vaults are - unusual but legal. -4. **Per-file vs full-vault check.** Post-check today uses - `mdvs check ` (full vault). Post-TODO-0172 this is ~8 ms - on a 1.6k-file vault, so full-vault is fine. A `mdvs check - --only ` for huge vaults is a follow-up if anyone hits a - limit. -5. **Exit code semantics.** Claude Code treats exit-2 from a hook - as "block + show stderr to the model". Confirm in the recipe - that this is the right signal for "agent should retry/fix" - (vs. exit-1 for a real hook error). Make explicit. + walk-up-to-`mdvs.toml` strategy handles it per-file. For nested vaults (a + `mdvs.toml` inside another `mdvs.toml`'s tree), the inner one wins. Document + the precedence; nested vaults are unusual but legal. +4. **Per-file vs full-vault check.** Post-check today uses `mdvs check ` + (full vault). Post-TODO-0172 this is ~8 ms on a 1.6k-file vault, so + full-vault is fine. A `mdvs check --only ` for huge vaults is a + follow-up if anyone hits a limit. +5. **Exit code semantics.** Claude Code treats exit-2 from a hook as "block + + show stderr to the model". Confirm in the recipe that this is the right + signal for "agent should retry/fix" (vs. exit-1 for a real hook error). Make + explicit. 6. **Edit handling.** - - **Post-check**: handled naturally — file on disk reflects - post-edit state, no special logic needed. Matcher includes - `Edit|MultiEdit`. - - **Pre-explain**: also natural — `file_path` from the Edit - payload is enough. - - **Pre-validate for Edit**: not in v1 of pre-validate. Would - require applying the patch to the current file content in - memory before validating. Tractable but more complex. v3+ - stretch if anyone asks. -7. **Performance budget.** Hook fires on every Write/Edit. Cost - is ~8 ms validation + a few ms for the hook script overhead. - Imperceptible at typical vault sizes. Worth a note in the - recipe for users with very large vaults. + - **Post-check**: handled naturally — file on disk reflects post-edit state, + no special logic needed. Matcher includes `Edit|MultiEdit`. + - **Pre-explain**: also natural — `file_path` from the Edit payload is + enough. + - **Pre-validate for Edit**: not in v1 of pre-validate. Would require + applying the patch to the current file content in memory before validating. + Tractable but more complex. v3+ stretch if anyone asks. +7. **Performance budget.** Hook fires on every Write/Edit. Cost is ~8 ms + validation + a few ms for the hook script overhead. Imperceptible at typical + vault sizes. Worth a note in the recipe for users with very large vaults. ## Verification - Recipe page renders cleanly in `mdbook build book`. -- All three hook scripts are `set -euo pipefail` clean and work on - macOS + Linux (smoke-test in CI if practical). -- End-to-end demo for v1: in a scratch vault with `mdvs init` run, - save a `.md` with an out-of-category `status` value via the - harness. Hook fires, `mdvs check` returns violations, agent sees - them in the next turn. -- Negative test: writing a `.md` file *outside* any vault triggers - no hook output (silent exit 0). +- All three hook scripts are `set -euo pipefail` clean and work on macOS + Linux + (smoke-test in CI if practical). +- End-to-end demo for v1: in a scratch vault with `mdvs init` run, save a `.md` + with an out-of-category `status` value via the harness. Hook fires, + `mdvs check` returns violations, agent sees them in the next turn. +- Negative test: writing a `.md` file _outside_ any vault triggers no hook + output (silent exit 0). - Negative test: writing a `.txt` file (not `.md`) triggers nothing. -- `crates/mdvs/Cargo.toml`'s `include = [...]` lists `hooks/` so - scripts ship with the bundle. +- `crates/mdvs/Cargo.toml`'s `include = [...]` lists `hooks/` so scripts ship + with the bundle. ## Out of scope -- A native mdvs-side "watch mode" that runs validation on file - changes (would be an alternative to the harness hook; different - use case — separate TODO if it ever comes up). -- Integration with non-Claude-Code harnesses beyond a short - pointer/footnote. Community can extend. -- Real-time validation feedback inside the agent's editor — that's - an LSP-style feature, much bigger surface, out of scope (but - noted as one of the use cases TODO-0188 enables). +- A native mdvs-side "watch mode" that runs validation on file changes (would be + an alternative to the harness hook; different use case — separate TODO if it + ever comes up). +- Integration with non-Claude-Code harnesses beyond a short pointer/footnote. + Community can extend. +- Real-time validation feedback inside the agent's editor — that's an LSP-style + feature, much bigger surface, out of scope (but noted as one of the use cases + TODO-0188 enables). - A `mdvs hooks install` subcommand. Deferred to a v2 follow-up. ## Impact -This is the second half of the agent-affordance story (paired with -TODO-0186 and TODO-0188). The combined loop becomes: +This is the second half of the agent-affordance story (paired with TODO-0186 and +TODO-0188). The combined loop becomes: -1. Agent decides to write a note → PreToolUse-explain fires → - agent has the schema in context. +1. Agent decides to write a note → PreToolUse-explain fires → agent has the + schema in context. 2. Agent composes content (correctly, on first try). 3. PreToolUse-validate fires → bad content blocked before write. -4. Write succeeds → PostToolUse-check fires → silent confirmation - that the file on disk is valid. +4. Write succeeds → PostToolUse-check fires → silent confirmation that the file + on disk is valid. -No back-and-forth, no manual reminders, no wasted writes. Each -mode is independently useful; the combined recipe is the gold -standard. +No back-and-forth, no manual reminders, no wasted writes. Each mode is +independently useful; the combined recipe is the gold standard. diff --git a/docs/spec/todos/TODO-0188.md b/docs/spec/todos/TODO-0188.md index fb02d7b..c67532e 100644 --- a/docs/spec/todos/TODO-0188.md +++ b/docs/spec/todos/TODO-0188.md @@ -13,29 +13,27 @@ related: [190] ## Problem -Today `mdvs check` reads files from disk and validates their -frontmatter. There's no way to ask "validate this proposed content -as if it lived at " without first writing the content to disk. +Today `mdvs check` reads files from disk and validates their frontmatter. +There's no way to ask "validate this proposed content as if it lived at " +without first writing the content to disk. That gap matters in three concrete settings: -1. **Pre-validate agent hooks** ([TODO-0187] v3) — a PreToolUse - hook wants to validate the agent's proposed file content before - the write reaches disk. Today the only way is to write a temp - file, run check, delete it. Awkward and racy. -2. **Editor / LSP integrations** — a future "mdvs LSP" or editor - plugin would underline frontmatter errors as the user types. - Today it would need to flush the buffer to disk on every - keystroke. -3. **CI lint of generated content** — a generator (script, agent, - bot) producing markdown wants to validate the artifact before - committing or shipping it. `printf | mdvs check --stdin path` is - the natural shape; today the only path is "write to temp, - check, delete." - -In all three cases the question is the same: "validate this byte -buffer as if it were the file at this path." The validation logic -is identical to the on-disk path; only the input source differs. +1. **Pre-validate agent hooks** ([TODO-0187] v3) — a PreToolUse hook wants to + validate the agent's proposed file content before the write reaches disk. + Today the only way is to write a temp file, run check, delete it. Awkward and + racy. +2. **Editor / LSP integrations** — a future "mdvs LSP" or editor plugin would + underline frontmatter errors as the user types. Today it would need to flush + the buffer to disk on every keystroke. +3. **CI lint of generated content** — a generator (script, agent, bot) producing + markdown wants to validate the artifact before committing or shipping it. + `printf | mdvs check --stdin path` is the natural shape; today the only path + is "write to temp, check, delete." + +In all three cases the question is the same: "validate this byte buffer as if it +were the file at this path." The validation logic is identical to the on-disk +path; only the input source differs. ## Goal @@ -46,92 +44,85 @@ mdvs check --stdin [--output pretty|markdown|json] ``` - Reads file content from stdin. -- Treats it as if it lived at `` (used to resolve - vault location via walk-up to `mdvs.toml`, and to evaluate - `allowed` / `required` globs against the path). +- Treats it as if it lived at `` (used to resolve vault location + via walk-up to `mdvs.toml`, and to evaluate `allowed` / `required` globs + against the path). - Runs the same `validate` pipeline as the on-disk path. -- Emits violations (or empty success) via the existing output - format (`pretty`, `markdown`, or `json`). -- Exits 0 on no violations, 1 on violations, 2 on errors (mirrors - current `mdvs check` semantics). +- Emits violations (or empty success) via the existing output format (`pretty`, + `markdown`, or `json`). +- Exits 0 on no violations, 1 on violations, 2 on errors (mirrors current + `mdvs check` semantics). -The `` is required because path-scoping is part of -the validation contract — the same content can be valid at one -path and invalid at another. The file at `` need not -exist. +The `` is required because path-scoping is part of the validation +contract — the same content can be valid at one path and invalid at another. The +file at `` need not exist. ## Approach -The factor is straightforward — the validation pipeline already -takes parsed frontmatter, not a file path: - -1. **Detect / parse frontmatter** — currently - `discover::scan::scan_file` reads from disk via - `std::fs::read_to_string`. Factor this into two functions: one - that takes bytes (the new "stdin" entry point) and one that - reads from disk and delegates (the existing entry point). Same - per-format dispatch (YAML / TOML / JSON via leading delimiter, - or forced via `[scan].frontmatter_format`). -2. **Resolve vault** — walk up from `` looking for - `mdvs.toml`. Same logic as TODO-0186's vault resolution; reuse - it (or factor into a shared helper). +The factor is straightforward — the validation pipeline already takes parsed +frontmatter, not a file path: + +1. **Detect / parse frontmatter** — currently `discover::scan::scan_file` reads + from disk via `std::fs::read_to_string`. Factor this into two functions: one + that takes bytes (the new "stdin" entry point) and one that reads from disk + and delegates (the existing entry point). Same per-format dispatch (YAML / + TOML / JSON via leading delimiter, or forced via + `[scan].frontmatter_format`). +2. **Resolve vault** — walk up from `` looking for `mdvs.toml`. + Same logic as TODO-0186's vault resolution; reuse it (or factor into a shared + helper). 3. **Run validate** — pass the parsed frontmatter through - `cmd::check::validate`, the same function the on-disk path uses. - `validate` takes parsed frontmatter + path + schema, so no - changes needed in the validator itself. -4. **Emit violations** — same `CommandResult` / `CommandOutput` - plumbing as on-disk check. + `cmd::check::validate`, the same function the on-disk path uses. `validate` + takes parsed frontmatter + path + schema, so no changes needed in the + validator itself. +4. **Emit violations** — same `CommandResult` / `CommandOutput` plumbing as + on-disk check. ### CLI shape -`--stdin ` is mutually exclusive with the positional -path arg of `mdvs check`. Clap should reject `mdvs check ./vault ---stdin notes/x.md`. +`--stdin ` is mutually exclusive with the positional path arg of +`mdvs check`. Clap should reject `mdvs check ./vault --stdin notes/x.md`. -If `--stdin` is given and stdin is a TTY (interactive use, no -piped content), error with a clear message rather than blocking. +If `--stdin` is given and stdin is a TTY (interactive use, no piped content), +error with a clear message rather than blocking. ### Multiple files via repeated invocations -The single-file shape is the v1. Batch use is `for f in ...; do -cat $f | mdvs check --stdin $f; done`. A `--stdin-list` flag that -accepts a stream of `\n\n---END---\n` -records is a v2 if anyone wants it. +The single-file shape is the v1. Batch use is +`for f in ...; do cat $f | mdvs check --stdin $f; done`. A `--stdin-list` flag +that accepts a stream of `\n\n---END---\n` records is a +v2 if anyone wants it. ## Design questions to settle before implementation -1. **Flag name** — `--stdin` is concise but a little generic. - Alternatives: `--from-stdin`, `--content-stdin`, `--virtual `. - I'd keep `--stdin ` — the positional argument - carries the "virtual" semantics. -2. **Should `--stdin` skip vault resolution?** If the agent already - knows which mdvs.toml applies (e.g., the hook caller resolved - it), an optional `--config ` could let the caller pass it - in directly, skipping the walk-up. Useful for tests + scripts. - Defer to v2. -3. **What about content without frontmatter?** If stdin is just - markdown body (no `---` / `+++` / `{...}` header), today - `mdvs check` would treat the on-disk file as a bare file and - apply the bare-file rules. Same behavior here — bare content is - valid unless a field is required at this path. -4. **Empty stdin** — error or treat as bare content? Treat as bare - content (consistent with on-disk: an empty file is bare). The - user can use `--stdin `. I'd keep + `--stdin ` — the positional argument carries the "virtual" + semantics. +2. **Should `--stdin` skip vault resolution?** If the agent already knows which + mdvs.toml applies (e.g., the hook caller resolved it), an optional + `--config ` could let the caller pass it in directly, skipping the + walk-up. Useful for tests + scripts. Defer to v2. +3. **What about content without frontmatter?** If stdin is just markdown body + (no `---` / `+++` / `{...}` header), today `mdvs check` would treat the + on-disk file as a bare file and apply the bare-file rules. Same behavior here + — bare content is valid unless a field is required at this path. +4. **Empty stdin** — error or treat as bare content? Treat as bare content + (consistent with on-disk: an empty file is bare). The user can use + `--stdin ` outside any vault → clean error message. @@ -140,37 +131,31 @@ records is a v2 if anyone wants it. ## Out of scope -- `--stdin-list` for multi-file validation in one invocation. - Single-file is enough for the agent / LSP cases; batch is just - a shell loop today. -- Validating partial / draft frontmatter (e.g., "I have title and - status filled in but no body yet"). Today's validator already - handles partial frontmatter — required-field violations surface - for the missing ones, which is the right behavior for "is this - draft valid?" checks. No special handling needed. -- A full LSP server. This TODO only adds the validate-from-content - primitive that an LSP would build on; the LSP itself is a much - larger surface (incremental parsing, document URI tracking, - diagnostics protocol). +- `--stdin-list` for multi-file validation in one invocation. Single-file is + enough for the agent / LSP cases; batch is just a shell loop today. +- Validating partial / draft frontmatter (e.g., "I have title and status filled + in but no body yet"). Today's validator already handles partial frontmatter — + required-field violations surface for the missing ones, which is the right + behavior for "is this draft valid?" checks. No special handling needed. +- A full LSP server. This TODO only adds the validate-from-content primitive + that an LSP would build on; the LSP itself is a much larger surface + (incremental parsing, document URI tracking, diagnostics protocol). ## Impact -Unblocks two follow-ups: the pre-validate hook in TODO-0187 (the -strongest form of agent-input gating) and any future editor / -LSP-style integration. Also useful in isolation for generator -scripts that want to lint output before writing. +Unblocks two follow-ups: the pre-validate hook in TODO-0187 (the strongest form +of agent-input gating) and any future editor / LSP-style integration. Also +useful in isolation for generator scripts that want to lint output before +writing. -The validation logic is already path-agnostic in spirit — this -TODO just plumbs the input boundary so callers can supply content -instead of a file path. Small surface change; meaningful -capability addition. +The validation logic is already path-agnostic in spirit — this TODO just plumbs +the input boundary so callers can supply content instead of a file path. Small +surface change; meaningful capability addition. ### Follow-up: pre-validate hook mode (v3) -[TODO-0190](TODO-0190.md) ships `mdvs scaffold hook` with v1 -(post-check) only. **v3 (pre-validate)** — the `PreToolUse` hook -that validates the proposed file content in-memory before the -write reaches disk, blocking bad writes entirely — needs this TODO -(`mdvs check --stdin`) to exist. Once 0188 lands, open a follow-up -TODO to add the v3 mode to the per-platform scaffolding hook -scripts. +[TODO-0190](TODO-0190.md) ships `mdvs scaffold hook` with v1 (post-check) only. +**v3 (pre-validate)** — the `PreToolUse` hook that validates the proposed file +content in-memory before the write reaches disk, blocking bad writes entirely — +needs this TODO (`mdvs check --stdin`) to exist. Once 0188 lands, open a +follow-up TODO to add the v3 mode to the per-platform scaffolding hook scripts. diff --git a/docs/spec/todos/TODO-0189.md b/docs/spec/todos/TODO-0189.md index f57725f..a668515 100644 --- a/docs/spec/todos/TODO-0189.md +++ b/docs/spec/todos/TODO-0189.md @@ -16,20 +16,34 @@ files_updated: ## Summary -The same 12-line `match (&cli.output, verbose) { ... }` block was duplicated nine times across `main.rs`, once per command. Adding any new `OutputFormat` variant required updating all nine call sites. Pulled the dispatch into a single method on `CommandResult` so adding a new format is now a two-touch change: one new enum variant in `output.rs` + one new match arm in `CommandResult::render`. +The same 12-line `match (&cli.output, verbose) { ... }` block was duplicated +nine times across `main.rs`, once per command. Adding any new `OutputFormat` +variant required updating all nine call sites. Pulled the dispatch into a single +method on `CommandResult` so adding a new format is now a two-touch change: one +new enum variant in `output.rs` + one new match arm in `CommandResult::render`. ## Resolution -Added `CommandResult::render(&self, format: &OutputFormat, verbose: bool) -> Result` on `step.rs` plus a private `compact_json` helper for the `Json + !verbose` branch. The method takes `&OutputFormat` (not owned), so call sites pass `&cli.output` with no clone. All nine sites in `main.rs` collapsed to: +Added +`CommandResult::render(&self, format: &OutputFormat, verbose: bool) -> Result` +on `step.rs` plus a private `compact_json` helper for the `Json + !verbose` +branch. The method takes `&OutputFormat` (not owned), so call sites pass +`&cli.output` with no clone. All nine sites in `main.rs` collapsed to: ```rust print!("{}", result.render(&cli.output, verbose)?); ``` -Five new unit tests in `step.rs` cover the four (format, verbose) combinations plus the JSON-compact error-fallback path. The `Pretty` and `Markdown` arms added in [[101]] reuse the same dispatch shape — confirming the two-touch claim. +Five new unit tests in `step.rs` cover the four (format, verbose) combinations +plus the JSON-compact error-fallback path. The `Pretty` and `Markdown` arms +added in [[101]] reuse the same dispatch shape — confirming the two-touch claim. -The opus pre-audit verified all nine call sites were byte-identical, exit-code logic stayed at the call site, and no module-dependency cycle was introduced (`step.rs` gained one-way imports of `OutputFormat` and `format_pretty`). +The opus pre-audit verified all nine call sites were byte-identical, exit-code +logic stayed at the call site, and no module-dependency cycle was introduced +(`step.rs` gained one-way imports of `OutputFormat` and `format_pretty`). -Net diff: `main.rs −108` lines, `step.rs +99` lines. Verified clippy clean and 817 unit tests pass. +Net diff: `main.rs −108` lines, `step.rs +99` lines. Verified clippy clean and +817 unit tests pass. -Cross-references: [[101]] (markdown output format) depends on this refactor as its prep step. +Cross-references: [[101]] (markdown output format) depends on this refactor as +its prep step. diff --git a/docs/spec/todos/TODO-0190.md b/docs/spec/todos/TODO-0190.md index eea8e71..c4a5f2a 100644 --- a/docs/spec/todos/TODO-0190.md +++ b/docs/spec/todos/TODO-0190.md @@ -44,41 +44,108 @@ files_updated: # TODO-0190: Design `mdvs scaffold` — unified agent-harness integration command surface -> **PIVOT 2026-06-22 (afternoon).** Hook logic moves into mdvs itself as `mdvs hook handle --platform --kind {validate|search-nudge}` rather than living in per-platform shell scripts. Driver: Windows support — the shell-based scaffolding shipped in commits `63311cf` + `01d8ff8` only runs on Mac/Linux. Pulling the logic into mdvs (already a cross-platform Rust binary) eliminates the OS dependency, drops the `jq` runtime dependency, and lets new platforms be added via a `platform.toml` config file with no Rust changes. See ["Pivot to cross-platform mdvs-internal hooks"](#pivot-to-cross-platform-mdvs-internal-hooks) below for the new architecture. The shell-script content from `63311cf` + `01d8ff8` is transitional reference material; the production implementation goes through the Rust path. +> **PIVOT 2026-06-22 (afternoon).** Hook logic moves into mdvs itself as +> `mdvs hook handle --platform --kind {validate|search-nudge}` rather +> than living in per-platform shell scripts. Driver: Windows support — the +> shell-based scaffolding shipped in commits `63311cf` + `01d8ff8` only runs on +> Mac/Linux. Pulling the logic into mdvs (already a cross-platform Rust binary) +> eliminates the OS dependency, drops the `jq` runtime dependency, and lets new +> platforms be added via a `platform.toml` config file with no Rust changes. See +> ["Pivot to cross-platform mdvs-internal hooks"](#pivot-to-cross-platform-mdvs-internal-hooks) +> below for the new architecture. The shell-script content from `63311cf` + +> `01d8ff8` is transitional reference material; the production implementation +> goes through the Rust path. ## Summary -Replace the narrow `mdvs skill` command with a broader `mdvs scaffold` surface that emits all three agent-harness integration artifacts — the bundled skill file, the project-rules snippet, and platform-specific hook configs — with `--platform` awareness for the parts that vary across harnesses (Claude Code, Codex, OpenCode, Cursor, Antigravity). Add `mdvs hook handle` as the cross-platform runtime for the hooks themselves: configured per platform via `scaffolding/platforms//platform.toml`, no shell scripts shipped, no `jq` dependency, works on Windows for free. - -This TODO captures the design conversation that happened on 2026-06-22 around the agent-harnesses recipe page; the recipe revealed that hand-written hook JSON, ad-hoc skill install paths, and a not-yet-existing snippet flag all want to live under a single, coherent command. Mid-implementation (commits `63311cf` + `01d8ff8`) the shell-script scaffolding showed it wouldn't work on Windows and the design pivoted to mdvs-internal hooks. +Replace the narrow `mdvs skill` command with a broader `mdvs scaffold` surface +that emits all three agent-harness integration artifacts — the bundled skill +file, the project-rules snippet, and platform-specific hook configs — with +`--platform` awareness for the parts that vary across harnesses (Claude Code, +Codex, OpenCode, Cursor, Antigravity). Add `mdvs hook handle` as the +cross-platform runtime for the hooks themselves: configured per platform via +`scaffolding/platforms//platform.toml`, no shell scripts shipped, no `jq` +dependency, works on Windows for free. + +This TODO captures the design conversation that happened on 2026-06-22 around +the agent-harnesses recipe page; the recipe revealed that hand-written hook +JSON, ad-hoc skill install paths, and a not-yet-existing snippet flag all want +to live under a single, coherent command. Mid-implementation (commits +`63311cf` + `01d8ff8`) the shell-script scaffolding showed it wouldn't work on +Windows and the design pivoted to mdvs-internal hooks. ## Resolution -Shipped on `feat/mdvs-wire` (PR #66), merged into `main`. The full command surface: - -- `mdvs scaffold skill --platform ` — writes a 368-line agent skill (content identical across platforms; install path varies) into the harness-native `*/skills/mdvs/SKILL.md` location. -- `mdvs scaffold snippet --platform ` — appends a rule snippet ("prefer `mdvs search` over `Grep`") to the harness's rules file (`CLAUDE.md`, `AGENTS.md`, etc.). -- `mdvs scaffold hook --platform ` — emits the harness's PostToolUse hook config wired to `mdvs hook handle`. Refuses with a pointer for OpenCode (TypeScript plugin API only — see `crates/mdvs/examples/opencode-plugin/mdvs-hooks.ts`) and Antigravity (user-level hooks only — documented in the recipe page). -- `mdvs hook handle --platform --kind {validate|search-nudge}` — cross-platform runtime that reads the harness's tool-call JSON on stdin, walks up to find `mdvs.toml`, runs `check` (validate) or formats a search nudge (search-nudge), and prints the platform-specific envelope. Silent on clean / outside-a-vault / hook-disabled — never interrupts the agent. - -**Per-platform shapes are template-driven.** `crates/mdvs/scaffolding//platform.toml` declares the JSON skeleton with `<>` placeholders (prune-on-None). Adding a sixth harness is a config edit, not a new Rust enum variant — confirmed during the design conversation as the right altitude. - -**Five platforms shipped.** `claude-code` (hooks verified end-to-end), `codex` (hook schema-correct per docs; firing status unclear in initial smoke test), `cursor` (hook schema-correct per docs; not observed firing in initial smoke test — needs investigation), `antigravity` (skill + snippet only — no project-level hooks upstream), `opencode` (skill + snippet only — TypeScript bridge plugin documented but did not fire in initial smoke). The Refractions personal vault was used as the install target for all five during development. The skill and snippet halves work everywhere; only Claude Code's hook half is known to work end-to-end. See `book/src/recipes/agent-harnesses.md` "What's tested" section for the current honest status of each. - -**Documentation.** New `book/src/recipes/agent-harnesses/` chapter (overview + one page per harness, alphabetical). First chapter in the Recipes section. `docs/spec/scaffolding.md` captures the architectural decisions (template-driven shapes, walk-up-silent behavior, refusal patterns) for future maintainers. - -**Subsumes [TODO-0187](TODO-0187.md)** (agent harness hook recipe) — the recipe page that 0187 proposed is part of what shipped here. - -**Defers** to follow-ups: TODO-0186 (`mdvs explain`, deferred to post-launch), TODO-0188 (`check --stdin`, deferred). The shipped hook surface assumes the simpler "post-edit validate" semantics; explain and stdin-mode are orthogonal additions. - -The design conversation captured below remains as the audit trail — including the mid-implementation pivot from shell scripts to mdvs-internal hooks — because the pivot's reasoning (Windows support, drop `jq` dependency, no per-platform shell variance) is load-bearing for anyone proposing a v2 design change. +Shipped on `feat/mdvs-wire` (PR #66), merged into `main`. The full command +surface: + +- `mdvs scaffold skill --platform ` — writes a 368-line agent skill + (content identical across platforms; install path varies) into the + harness-native `*/skills/mdvs/SKILL.md` location. +- `mdvs scaffold snippet --platform ` — appends a rule snippet ("prefer + `mdvs search` over `Grep`") to the harness's rules file (`CLAUDE.md`, + `AGENTS.md`, etc.). +- `mdvs scaffold hook --platform ` — emits the harness's PostToolUse hook + config wired to `mdvs hook handle`. Refuses with a pointer for OpenCode + (TypeScript plugin API only — see + `crates/mdvs/examples/opencode-plugin/mdvs-hooks.ts`) and Antigravity + (user-level hooks only — documented in the recipe page). +- `mdvs hook handle --platform --kind {validate|search-nudge}` — + cross-platform runtime that reads the harness's tool-call JSON on stdin, walks + up to find `mdvs.toml`, runs `check` (validate) or formats a search nudge + (search-nudge), and prints the platform-specific envelope. Silent on clean / + outside-a-vault / hook-disabled — never interrupts the agent. + +**Per-platform shapes are template-driven.** +`crates/mdvs/scaffolding//platform.toml` declares the JSON skeleton +with `<>` placeholders (prune-on-None). Adding a sixth harness is a +config edit, not a new Rust enum variant — confirmed during the design +conversation as the right altitude. + +**Five platforms shipped.** `claude-code` (hooks verified end-to-end), `codex` +(hook schema-correct per docs; firing status unclear in initial smoke test), +`cursor` (hook schema-correct per docs; not observed firing in initial smoke +test — needs investigation), `antigravity` (skill + snippet only — no +project-level hooks upstream), `opencode` (skill + snippet only — TypeScript +bridge plugin documented but did not fire in initial smoke). The Refractions +personal vault was used as the install target for all five during development. +The skill and snippet halves work everywhere; only Claude Code's hook half is +known to work end-to-end. See `book/src/recipes/agent-harnesses.md` "What's +tested" section for the current honest status of each. + +**Documentation.** New `book/src/recipes/agent-harnesses/` chapter (overview + +one page per harness, alphabetical). First chapter in the Recipes section. +`docs/spec/scaffolding.md` captures the architectural decisions (template-driven +shapes, walk-up-silent behavior, refusal patterns) for future maintainers. + +**Subsumes [TODO-0187](TODO-0187.md)** (agent harness hook recipe) — the recipe +page that 0187 proposed is part of what shipped here. + +**Defers** to follow-ups: TODO-0186 (`mdvs explain`, deferred to post-launch), +TODO-0188 (`check --stdin`, deferred). The shipped hook surface assumes the +simpler "post-edit validate" semantics; explain and stdin-mode are orthogonal +additions. + +The design conversation captured below remains as the audit trail — including +the mid-implementation pivot from shell scripts to mdvs-internal hooks — because +the pivot's reasoning (Windows support, drop `jq` dependency, no per-platform +shell variance) is load-bearing for anyone proposing a v2 design change. ## Background Two things triggered this: -1. While writing `book/src/recipes/agentic-harnesses-and-agentic-ides.md` we realized the recipe was teaching users to hand-write per-harness hook JSON blocks. That's exactly the kind of platform-specific glue a tool should emit, not the user. -2. We discovered the current validation-hook example was silently broken: the hook's stdout goes to Claude Code's debug log (per [hooks reference](https://code.claude.com/docs/en/hooks)), not to the model. For the agent to see violation feedback, the hook must emit a JSON envelope (`{hookSpecificOutput: {hookEventName: "PostToolUse", additionalContext: "..."}}`) and exit 0. Writing that inline as a `jq` pipeline in every user's `settings.json` is unreasonable; mdvs should emit it. +1. While writing `book/src/recipes/agentic-harnesses-and-agentic-ides.md` we + realized the recipe was teaching users to hand-write per-harness hook JSON + blocks. That's exactly the kind of platform-specific glue a tool should emit, + not the user. +2. We discovered the current validation-hook example was silently broken: the + hook's stdout goes to Claude Code's debug log (per + [hooks reference](https://code.claude.com/docs/en/hooks)), not to the model. + For the agent to see violation feedback, the hook must emit a JSON envelope + (`{hookSpecificOutput: {hookEventName: "PostToolUse", additionalContext: "..."}}`) + and exit 0. Writing that inline as a `jq` pipeline in every user's + `settings.json` is unreasonable; mdvs should emit it. ## Proposed surface @@ -88,32 +155,58 @@ mdvs scaffold snippet [--platform ] # default: harness-neutral AGENTS.m mdvs scaffold hook --platform # platform required (no sensible default) ``` -Aliases like `claude` / `cc`, `codex`, `oc`, `cursor`, `antigravity` for ergonomics. +Aliases like `claude` / `cc`, `codex`, `oc`, `cursor`, `antigravity` for +ergonomics. ### What each subcommand emits -- **`scaffold skill`** — the bundled `SKILL.md` content (refreshed from the existing `crates/mdvs/skills/mdvs/SKILL.md`, 347 lines). Platform variants differ only in install-path hints in surrounding help text; the SKILL.md body itself follows the [Agent Skills open standard](https://agentskills.io) and is the same across harnesses. -- **`scaffold snippet`** — a short ~10–15 line block to paste into the harness's project-rules file (`AGENTS.md` for most; `CLAUDE.md` for Claude Code). Content covers: KB presence, `mdvs search` preference over Grep/Glob, `mdvs check` validation contract, the schema-evolution "warning, not block" rule. -- **`scaffold hook`** — emits both the `.claude/settings.json`-style config block AND the shell-script body for the validate-on-write and search-nudge hooks. Format differs per harness: Claude Code JSON, Codex JSON/TOML, Cursor JSON with camelCase event names. OpenCode and Antigravity refuse with a pointer to the recipe page (different surfaces / undocumented). +- **`scaffold skill`** — the bundled `SKILL.md` content (refreshed from the + existing `crates/mdvs/skills/mdvs/SKILL.md`, 347 lines). Platform variants + differ only in install-path hints in surrounding help text; the SKILL.md body + itself follows the [Agent Skills open standard](https://agentskills.io) and is + the same across harnesses. +- **`scaffold snippet`** — a short ~10–15 line block to paste into the harness's + project-rules file (`AGENTS.md` for most; `CLAUDE.md` for Claude Code). + Content covers: KB presence, `mdvs search` preference over Grep/Glob, + `mdvs check` validation contract, the schema-evolution "warning, not block" + rule. +- **`scaffold hook`** — emits both the `.claude/settings.json`-style config + block AND the shell-script body for the validate-on-write and search-nudge + hooks. Format differs per harness: Claude Code JSON, Codex JSON/TOML, Cursor + JSON with camelCase event names. OpenCode and Antigravity refuse with a + pointer to the recipe page (different surfaces / undocumented). ### Envelope wrapping lives in the shell scripts, NOT in mdvs -An earlier draft of this TODO proposed adding per-platform `OutputFormat` variants (`claude-code-hook`, `codex-hook`, etc.) that would emit the hook-output JSON envelope directly from `mdvs check`. That approach is rejected because: +An earlier draft of this TODO proposed adding per-platform `OutputFormat` +variants (`claude-code-hook`, `codex-hook`, etc.) that would emit the +hook-output JSON envelope directly from `mdvs check`. That approach is rejected +because: -- mdvs would have to track each harness's envelope schema, which changes on the harness's release cadence; -- mdvs's CLI surface would gain platform-specific format names that aren't really mdvs concerns; -- per-platform glue belongs in per-platform files — the bundled hook scripts — not in mdvs Rust. +- mdvs would have to track each harness's envelope schema, which changes on the + harness's release cadence; +- mdvs's CLI surface would gain platform-specific format names that aren't + really mdvs concerns; +- per-platform glue belongs in per-platform files — the bundled hook scripts — + not in mdvs Rust. Instead: -- **`mdvs check` stays harness-agnostic.** It emits `markdown` or `json` per existing `OutputFormat`. No new variants. -- **Per-platform shell logic wraps mdvs's output in the platform's envelope.** When a harness changes its envelope schema, the update is contained to that platform's shell, not in mdvs Rust. +- **`mdvs check` stays harness-agnostic.** It emits `markdown` or `json` per + existing `OutputFormat`. No new variants. +- **Per-platform shell logic wraps mdvs's output in the platform's envelope.** + When a harness changes its envelope schema, the update is contained to that + platform's shell, not in mdvs Rust. ### Concrete shell sketches -Two distinct hook shapes, both for Claude Code's PostToolUse (Codex follows the same pattern unchanged; Cursor swaps `PostToolUse` → `postToolUse` in the envelope and the matcher): +Two distinct hook shapes, both for Claude Code's PostToolUse (Codex follows the +same pattern unchanged; Cursor swaps `PostToolUse` → `postToolUse` in the +envelope and the matcher): -**Validate-on-write hook** — separate `.sh` file, captures mdvs output, wraps it in the envelope when non-empty. Lives at `scaffolding/hooks/claude-code/validate.sh`: +**Validate-on-write hook** — separate `.sh` file, captures mdvs output, wraps it +in the envelope when non-empty. Lives at +`scaffolding/hooks/claude-code/validate.sh`: ```bash #!/usr/bin/env bash @@ -141,7 +234,8 @@ The wrapping `.claude/settings.json` just calls it: } ``` -**Search-nudge hook** — inline in `settings.json`, pure literal envelope, no captured output: +**Search-nudge hook** — inline in `settings.json`, pure literal envelope, no +captured output: ```json { @@ -153,123 +247,193 @@ The wrapping `.claude/settings.json` just calls it: } ``` -Both hooks exit 0 unconditionally; the harness treats the JSON envelope on stdout as "additional context the model should see" — the non-blocking warning path. +Both hooks exit 0 unconditionally; the harness treats the JSON envelope on +stdout as "additional context the model should see" — the non-blocking warning +path. ### Per-artifact variants — what actually differs per platform Not every artifact needs platform-specific content. Concrete count: -| Artifact | Variants | What differs | What stays universal | -|---|---|---|---| -| `SKILL.md` | **1** | Suggested install path comment in help text | Body content — the Agent Skills standard is shared across all five harnesses; the SKILL.md teaches the agent what mdvs is and when to call it, which is harness-agnostic. | -| Snippet (project-rules) | **2** | (a) Plain markdown for `CLAUDE.md` / `AGENTS.md`, (b) `.mdc`-wrapped (frontmatter with `alwaysApply: true`) for Cursor `.cursor/rules/mdvs.mdc`. | Body instructions to the agent — same everywhere. | -| Validate-on-write hook | **3** | Settings file path (`.claude/settings.json` vs `.codex/hooks.json` vs `.cursor/hooks.json`); event-name capitalization (`PostToolUse` vs `postToolUse`); tool matcher names (`Edit\|Write\|MultiEdit` vs equivalents). | The walk-up-to-`mdvs.toml` logic; the `mdvs check --output markdown` invocation; the empty-output-means-silent contract. The `.sh` body is ~95% identical across platforms — only the envelope's event-name string differs. | -| Search-nudge hook | **3** | Same platform-specific surface as validate hook, embedded inline in the settings file. | The case-match against `grep`/`rg`/`find`/`fd`/etc.; the KB-path scoping; the literal-envelope echo pattern. | +| Artifact | Variants | What differs | What stays universal | +| ----------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `SKILL.md` | **1** | Suggested install path comment in help text | Body content — the Agent Skills standard is shared across all five harnesses; the SKILL.md teaches the agent what mdvs is and when to call it, which is harness-agnostic. | +| Snippet (project-rules) | **2** | (a) Plain markdown for `CLAUDE.md` / `AGENTS.md`, (b) `.mdc`-wrapped (frontmatter with `alwaysApply: true`) for Cursor `.cursor/rules/mdvs.mdc`. | Body instructions to the agent — same everywhere. | +| Validate-on-write hook | **3** | Settings file path (`.claude/settings.json` vs `.codex/hooks.json` vs `.cursor/hooks.json`); event-name capitalization (`PostToolUse` vs `postToolUse`); tool matcher names (`Edit\|Write\|MultiEdit` vs equivalents). | The walk-up-to-`mdvs.toml` logic; the `mdvs check --output markdown` invocation; the empty-output-means-silent contract. The `.sh` body is ~95% identical across platforms — only the envelope's event-name string differs. | +| Search-nudge hook | **3** | Same platform-specific surface as validate hook, embedded inline in the settings file. | The case-match against `grep`/`rg`/`find`/`fd`/etc.; the KB-path scoping; the literal-envelope echo pattern. | -So `mdvs scaffold skill` and `mdvs scaffold snippet` have very small platform surface area. `mdvs scaffold hook` is where the per-platform code lives — three variants covering Claude Code, Codex, Cursor. +So `mdvs scaffold skill` and `mdvs scaffold snippet` have very small platform +surface area. `mdvs scaffold hook` is where the per-platform code lives — three +variants covering Claude Code, Codex, Cursor. ### `mdvs check` operates on the vault, not on a file -`mdvs check [PATH]` takes a directory containing `mdvs.toml`, not a file path. There is no per-file validation mode today (that would need [TODO-0188](TODO-0188.md) `mdvs check --stdin`). The hook flow is therefore: +`mdvs check [PATH]` takes a directory containing `mdvs.toml`, not a file path. +There is no per-file validation mode today (that would need +[TODO-0188](TODO-0188.md) `mdvs check --stdin`). The hook flow is therefore: 1. Hook fires on `Edit` / `Write`. 2. Read `file_path` from the stdin JSON. -3. Walk up the file's parent directories to find `mdvs.toml` — this is the vault root. +3. Walk up the file's parent directories to find `mdvs.toml` — this is the vault + root. 4. Run `mdvs check --output markdown`. 5. Wrap the output in the platform's envelope (shell script's job). 6. Exit 0. ## Decisions (Step 1 complete) -Twelve open design questions surfaced during the 2026-06-22 design conversation; all settled. Notes below for the record. - -1. ~~**Name.**~~ **Decided: `scaffold`.** The verb specifically means "generate boilerplate," which fits exactly. Directory under the crate is `scaffolding/` (singular mass noun, like `documentation/`). Alternatives considered: `wire` (shorter but more abstract), `adapter`, `agent`. -2. ~~**Subcommand order.**~~ **Decided: verb-object.** `scaffold skill | snippet | hook` — reads as an install command. -3. ~~**Hook output shape.**~~ **Decided: stdout-only in v1.** `scaffold hook` prints one big text block with clearly-delimited config + script sections, each prefixed by header comments explaining destination paths. `--write` flag deferred to v2 once paths and behavior are stable. -4. ~~`mdvs check --output ` naming.~~ **Dropped** — envelope wrapping moved to per-platform shell scripts (see "Envelope wrapping lives in the shell scripts, NOT in mdvs" above). -5. ~~**Relationship to [TODO-0187](TODO-0187.md).**~~ **Decided: subsume.** 0187's three composable hook modes (post-check / pre-explain / pre-validate) carry over as v1/v2/v3 of `scaffold hook`. 0190 ships **v1 (post-check) only**; v2 and v3 unlock when [TODO-0186](TODO-0186.md) and [TODO-0188](TODO-0188.md) land respectively, and are tracked in those TODOs' follow-up notes. 0187's exit-2/stderr output is replaced by the JSON envelope + exit 0 path. 0187 gets marked `subsumed_by: 190` when 0190 v1 ships. -6. ~~**Pre-1.0 migration.**~~ **Decided: hard rename.** `mdvs skill` is removed entirely; `mdvs scaffold skill` replaces it. No alias surface. Help text for the old name (if a stub is kept transiently) errors with a pointer. -7. ~~**Cursor split.**~~ **Decided: emit AGENTS.md form by default; the `.mdc` variant ships as a second template under `scaffolding/snippet/cursor-rules.mdc`** (selected by `--platform cursor` with optional `--target cursor-rules`). Body content identical; only the `.mdc` frontmatter wrapper differs. -8. ~~**OpenCode hooks.**~~ **Decided: refuse with pointer in v1.** OpenCode's hook surface is a TypeScript plugin API ([source](https://opencode.ai/docs/agents/)); no shell-command config. `scaffold hook --platform opencode` exits with a message pointing at the recipe page section that describes the community shell-hook package. Re-evaluate when/if OpenCode adds a first-class shell-hook config. -9. ~~**Antigravity hooks.**~~ **Decided: refuse with pointer in v1.** Upstream documentation is incomplete; inherited Gemini CLI hook docs use different event names (`BeforeTool` / `AfterTool`) but Antigravity's post-rebrand schema isn't published. Skill + snippet work on Antigravity (verified via [Google's authoring codelab](https://codelabs.developers.google.com/getting-started-with-antigravity-skills)); hooks await documentation. -10. ~~**Auto-detect platform.**~~ **Decided: no in v1.** Explicit `--platform` for every invocation. Auto-detect from cwd (`.claude/`, `.codex/`, etc.) is v2 work. -11. ~~**Inline shell vs separate script file.**~~ **Decided (initial): separate `.sh` for validate, inline for search-nudge.** **Re-decided (pivot 2026-06-22): neither — both move into mdvs as `mdvs hook handle`.** See [Pivot section](#pivot-to-cross-platform-mdvs-internal-hooks). The shell approach was reversed because it couldn't ship on Windows. -12. ~~**stdin parser choice.**~~ **Decided (initial): `jq`.** **Made moot by the pivot:** mdvs reads stdin JSON natively via serde, no external parser needed. The skill content references to `jq` are now historical / pedagogical (still useful for users who want to understand the contract). +Twelve open design questions surfaced during the 2026-06-22 design conversation; +all settled. Notes below for the record. + +1. ~~**Name.**~~ **Decided: `scaffold`.** The verb specifically means "generate + boilerplate," which fits exactly. Directory under the crate is `scaffolding/` + (singular mass noun, like `documentation/`). Alternatives considered: `wire` + (shorter but more abstract), `adapter`, `agent`. +2. ~~**Subcommand order.**~~ **Decided: verb-object.** + `scaffold skill | snippet | hook` — reads as an install command. +3. ~~**Hook output shape.**~~ **Decided: stdout-only in v1.** `scaffold hook` + prints one big text block with clearly-delimited config + script sections, + each prefixed by header comments explaining destination paths. `--write` flag + deferred to v2 once paths and behavior are stable. +4. ~~`mdvs check --output ` naming.~~ **Dropped** — envelope wrapping + moved to per-platform shell scripts (see "Envelope wrapping lives in the + shell scripts, NOT in mdvs" above). +5. ~~**Relationship to [TODO-0187](TODO-0187.md).**~~ **Decided: subsume.** + 0187's three composable hook modes (post-check / pre-explain / pre-validate) + carry over as v1/v2/v3 of `scaffold hook`. 0190 ships **v1 (post-check) + only**; v2 and v3 unlock when [TODO-0186](TODO-0186.md) and + [TODO-0188](TODO-0188.md) land respectively, and are tracked in those TODOs' + follow-up notes. 0187's exit-2/stderr output is replaced by the JSON + envelope + exit 0 path. 0187 gets marked `subsumed_by: 190` when 0190 v1 + ships. +6. ~~**Pre-1.0 migration.**~~ **Decided: hard rename.** `mdvs skill` is removed + entirely; `mdvs scaffold skill` replaces it. No alias surface. Help text for + the old name (if a stub is kept transiently) errors with a pointer. +7. ~~**Cursor split.**~~ **Decided: emit AGENTS.md form by default; the `.mdc` + variant ships as a second template under + `scaffolding/snippet/cursor-rules.mdc`** (selected by `--platform cursor` + with optional `--target cursor-rules`). Body content identical; only the + `.mdc` frontmatter wrapper differs. +8. ~~**OpenCode hooks.**~~ **Decided: refuse with pointer in v1.** OpenCode's + hook surface is a TypeScript plugin API + ([source](https://opencode.ai/docs/agents/)); no shell-command config. + `scaffold hook --platform opencode` exits with a message pointing at the + recipe page section that describes the community shell-hook package. + Re-evaluate when/if OpenCode adds a first-class shell-hook config. +9. ~~**Antigravity hooks.**~~ **Decided: refuse with pointer in v1.** Upstream + documentation is incomplete; inherited Gemini CLI hook docs use different + event names (`BeforeTool` / `AfterTool`) but Antigravity's post-rebrand + schema isn't published. Skill + snippet work on Antigravity (verified via + [Google's authoring codelab](https://codelabs.developers.google.com/getting-started-with-antigravity-skills)); + hooks await documentation. +10. ~~**Auto-detect platform.**~~ **Decided: no in v1.** Explicit `--platform` + for every invocation. Auto-detect from cwd (`.claude/`, `.codex/`, etc.) is + v2 work. +11. ~~**Inline shell vs separate script file.**~~ **Decided (initial): separate + `.sh` for validate, inline for search-nudge.** **Re-decided (pivot + 2026-06-22): neither — both move into mdvs as `mdvs hook handle`.** See + [Pivot section](#pivot-to-cross-platform-mdvs-internal-hooks). The shell + approach was reversed because it couldn't ship on Windows. +12. ~~**stdin parser choice.**~~ **Decided (initial): `jq`.** **Made moot by the + pivot:** mdvs reads stdin JSON natively via serde, no external parser + needed. The skill content references to `jq` are now historical / + pedagogical (still useful for users who want to understand the contract). ## Pivot to cross-platform mdvs-internal hooks -**When:** 2026-06-22, mid-Step-3, after shipping `63311cf` (Claude Code .sh) and `01d8ff8` (Codex + Cursor .sh + search-nudge extraction + example_kb dogfooding). +**When:** 2026-06-22, mid-Step-3, after shipping `63311cf` (Claude Code .sh) and +`01d8ff8` (Codex + Cursor .sh + search-nudge extraction + example_kb +dogfooding). -**Why:** Everything shipped is POSIX shell, requires `jq` on PATH, and won't run on Windows. mdvs is a cross-platform Rust binary; the hook glue should be too. The architectural argument that drove decision #4 ("envelope wrapping in shell, not mdvs") was based on the assumption that envelope schemas change per-harness on a fast cadence — in practice they don't (Claude/Codex/Cursor all converged on `{hookSpecificOutput: {hookEventName, additionalContext}}` with only the event-name capitalization differing). The cross-platform benefit outweighs the keep-mdvs-thin argument. +**Why:** Everything shipped is POSIX shell, requires `jq` on PATH, and won't run +on Windows. mdvs is a cross-platform Rust binary; the hook glue should be too. +The architectural argument that drove decision #4 ("envelope wrapping in shell, +not mdvs") was based on the assumption that envelope schemas change per-harness +on a fast cadence — in practice they don't (Claude/Codex/Cursor all converged on +`{hookSpecificOutput: {hookEventName, additionalContext}}` with only the +event-name capitalization differing). The cross-platform benefit outweighs the +keep-mdvs-thin argument. ### New architecture Three changes: -1. **`mdvs hook handle --platform --kind `** — new subcommand. Reads stdin JSON, runs the same logic the shell scripts did (walk-up to `mdvs.toml`, run `mdvs check` internally for validate / pattern-match command for search-nudge, wrap output in the per-platform envelope), prints to stdout, exits 0. All cross-platform Rust. No `jq`. No shell. - -2. **Per-platform behaviour is data, not code.** `scaffolding/platforms//platform.toml` declares everything that varies between harnesses: - - ```toml - [meta] - name = "claude-code" - display_name = "Claude Code" - - [skill] - install_path = ".claude/skills/mdvs/SKILL.md" - - [snippet] - target_file = "CLAUDE.md" - body = "agents-md" # key into scaffolding/snippet/ - - [hooks] - config_path = ".claude/settings.json" - config_format = "json" # or "toml" for Codex's [hooks] table form - event_name_validate = "PostToolUse" - event_name_search = "PostToolUse" - matcher_validate = "Edit|Write|MultiEdit" - matcher_search = "Bash" - - [hooks.envelope] - template = ''' - { - "hookSpecificOutput": { - "hookEventName": "$event_name", - "additionalContext": $msg - }, - "systemMessage": $user_msg - } - ''' - - [hooks.stdin_paths] - file_path = ".tool_input.file_path" - command = ".tool_input.command" - cwd = ".cwd" - ``` - - Adding a new harness = adding a new `platform.toml`. No Rust changes, no release for end users. - -3. **`scaffolding/` becomes config-only.** No more `.sh` files under `scaffolding/hooks//`. The bundled tree at ship time: - - ``` - scaffolding/ - ├── skill/SKILL.md ← universal - ├── snippet/ - │ ├── agents-md.md ← universal body - │ └── cursor-rules.mdc ← Cursor frontmatter wrap - └── platforms/ - ├── claude-code/platform.toml - ├── codex/platform.toml - ├── cursor/platform.toml - ├── opencode/platform.toml ← skill+snippet only, no [hooks] - └── antigravity/platform.toml ← same - ``` - - Bundled into the binary via `include_dir!` at build time. Loaded as `Platform` structs at runtime (not enums, not `dyn Trait` — plain data, see "Rust shape" below). +1. **`mdvs hook handle --platform --kind `** — new + subcommand. Reads stdin JSON, runs the same logic the shell scripts did + (walk-up to `mdvs.toml`, run `mdvs check` internally for validate / + pattern-match command for search-nudge, wrap output in the per-platform + envelope), prints to stdout, exits 0. All cross-platform Rust. No `jq`. No + shell. + +2. **Per-platform behaviour is data, not code.** + `scaffolding/platforms//platform.toml` declares everything that varies + between harnesses: + + ```toml + [meta] + name = "claude-code" + display_name = "Claude Code" + + [skill] + install_path = ".claude/skills/mdvs/SKILL.md" + + [snippet] + target_file = "CLAUDE.md" + body = "agents-md" # key into scaffolding/snippet/ + + [hooks] + config_path = ".claude/settings.json" + config_format = "json" # or "toml" for Codex's [hooks] table form + event_name_validate = "PostToolUse" + event_name_search = "PostToolUse" + matcher_validate = "Edit|Write|MultiEdit" + matcher_search = "Bash" + + [hooks.envelope] + template = ''' + { + "hookSpecificOutput": { + "hookEventName": "$event_name", + "additionalContext": $msg + }, + "systemMessage": $user_msg + } + ''' + + [hooks.stdin_paths] + file_path = ".tool_input.file_path" + command = ".tool_input.command" + cwd = ".cwd" + ``` + + Adding a new harness = adding a new `platform.toml`. No Rust changes, no + release for end users. + +3. **`scaffolding/` becomes config-only.** No more `.sh` files under + `scaffolding/hooks//`. The bundled tree at ship time: + + ``` + scaffolding/ + ├── skill/SKILL.md ← universal + ├── snippet/ + │ ├── agents-md.md ← universal body + │ └── cursor-rules.mdc ← Cursor frontmatter wrap + └── platforms/ + ├── claude-code/platform.toml + ├── codex/platform.toml + ├── cursor/platform.toml + ├── opencode/platform.toml ← skill+snippet only, no [hooks] + └── antigravity/platform.toml ← same + ``` + + Bundled into the binary via `include_dir!` at build time. Loaded as + `Platform` structs at runtime (not enums, not `dyn Trait` — plain data, see + "Rust shape" below). ### What the settings.json / hooks.json templates emit -After pivot, `mdvs scaffold hook --platform claude-code` emits a config that calls `mdvs` directly — no shell wrapper, no script files: +After pivot, `mdvs scaffold hook --platform claude-code` emits a config that +calls `mdvs` directly — no shell wrapper, no script files: ```json { @@ -286,7 +450,8 @@ After pivot, `mdvs scaffold hook --platform claude-code` emits a config that cal } ``` -That's it. Users only need `mdvs` on PATH (which they already do for everything else). No `jq`, no `.sh` files, works the same on every OS. +That's it. Users only need `mdvs` on PATH (which they already do for everything +else). No `jq`, no `.sh` files, works the same on every OS. ### Rust shape (plain struct, no enum, no `dyn Trait`) @@ -312,146 +477,246 @@ pub fn list_platforms() -> Vec { } ``` -No enum because platforms aren't a fixed set known at compile time — the whole point of the config-driven design is that new platforms come from new toml files. The project's "enum dispatch, no `dyn Trait`" rule still applies elsewhere (backends, embedders, value stages — concerns where finiteness genuinely matters); platforms aren't one of those. +No enum because platforms aren't a fixed set known at compile time — the whole +point of the config-driven design is that new platforms come from new toml +files. The project's "enum dispatch, no `dyn Trait`" rule still applies +elsewhere (backends, embedders, value stages — concerns where finiteness +genuinely matters); platforms aren't one of those. ### What this supersedes / preserves from the work already shipped -- **Superseded:** `scaffolding/hooks//{validate,search-nudge}.sh` and `{settings,hooks}.json` templates from commits `63311cf` + `01d8ff8`. The shell scripts get deleted; the JSON templates get regenerated to call `mdvs hook handle`. -- **Preserved:** `scaffolding/skill/SKILL.md` (universal, unchanged). `scaffolding/snippet/{agents-md.md,cursor-rules.mdc}` (universal, unchanged). The `example_kb/AGENTS.md` and `CLAUDE.md` symlink (unchanged). -- **Reshaped:** `example_kb/.{claude,codex,cursor}/` get cleaned up — no more `.sh` symlinks under `hooks/`, just config files calling `mdvs hook handle`. `example_kb/.agents/skills/` stays as-is (cross-harness skill path). +- **Superseded:** `scaffolding/hooks//{validate,search-nudge}.sh` and + `{settings,hooks}.json` templates from commits `63311cf` + `01d8ff8`. The + shell scripts get deleted; the JSON templates get regenerated to call + `mdvs hook handle`. +- **Preserved:** `scaffolding/skill/SKILL.md` (universal, unchanged). + `scaffolding/snippet/{agents-md.md,cursor-rules.mdc}` (universal, unchanged). + The `example_kb/AGENTS.md` and `CLAUDE.md` symlink (unchanged). +- **Reshaped:** `example_kb/.{claude,codex,cursor}/` get cleaned up — no more + `.sh` symlinks under `hooks/`, just config files calling `mdvs hook handle`. + `example_kb/.agents/skills/` stays as-is (cross-harness skill path). ### Why the existing scaffolding scripts are not lost work -They documented the exact shell logic the new Rust subcommand needs to implement. They're the executable spec for `mdvs hook handle`. The Rust impl is a direct port: +They documented the exact shell logic the new Rust subcommand needs to +implement. They're the executable spec for `mdvs hook handle`. The Rust impl is +a direct port: - Read stdin JSON → `serde_json::from_reader(stdin())` - Walk up to `mdvs.toml` → loop with `std::fs::metadata` -- Run `mdvs check` → call the existing `cmd::check` module directly (no subprocess) +- Run `mdvs check` → call the existing `cmd::check` module directly (no + subprocess) - Wrap in envelope → templated string substitution per `platform.toml` - Emit to stdout → `println!` - Exit 0 ## Platforms — confirmed conventions (research summary) -| Harness | Skill path | Snippet file | Hook config | Hook events | Hook input | Hook output for "model context" | -|---|---|---|---|---|---|---| -| Claude Code | `.claude/skills//SKILL.md` | `CLAUDE.md` | `.claude/settings.json` | `PreToolUse`, `PostToolUse` | stdin JSON | JSON envelope `{hookSpecificOutput: {hookEventName, additionalContext}}` | -| Codex | `.agents/skills//SKILL.md` | `AGENTS.md` (or `AGENTS.override.md`) | `.codex/hooks.json` or `[hooks]` in `config.toml` | `PreToolUse`, `PostToolUse` | stdin JSON | JSON envelope, same shape as Claude Code | -| OpenCode | `.opencode/skills/`, `.claude/skills/`, `.agents/skills/` (all read) | `AGENTS.md` | TypeScript plugin API (`tool.execute.before/after`) — no shell-config file | n/a (plugin) | n/a (plugin) | n/a (plugin) | -| Cursor | `.cursor/skills/`, `.agents/skills/`, `.claude/skills/`, `.codex/skills/` (all read) | `AGENTS.md` or `.cursor/rules/*.mdc` | `.cursor/hooks.json` | camelCase: `postToolUse` etc. | stdin JSON | JSON envelope (verify exact shape during impl) | -| Antigravity | `.agents/skills//SKILL.md` | `AGENTS.md` | undocumented at time of writing | unknown | unknown | unknown | +| Harness | Skill path | Snippet file | Hook config | Hook events | Hook input | Hook output for "model context" | +| ----------- | ------------------------------------------------------------------------------------ | ------------------------------------- | -------------------------------------------------------------------------- | ----------------------------- | ------------ | ------------------------------------------------------------------------ | +| Claude Code | `.claude/skills//SKILL.md` | `CLAUDE.md` | `.claude/settings.json` | `PreToolUse`, `PostToolUse` | stdin JSON | JSON envelope `{hookSpecificOutput: {hookEventName, additionalContext}}` | +| Codex | `.agents/skills//SKILL.md` | `AGENTS.md` (or `AGENTS.override.md`) | `.codex/hooks.json` or `[hooks]` in `config.toml` | `PreToolUse`, `PostToolUse` | stdin JSON | JSON envelope, same shape as Claude Code | +| OpenCode | `.opencode/skills/`, `.claude/skills/`, `.agents/skills/` (all read) | `AGENTS.md` | TypeScript plugin API (`tool.execute.before/after`) — no shell-config file | n/a (plugin) | n/a (plugin) | n/a (plugin) | +| Cursor | `.cursor/skills/`, `.agents/skills/`, `.claude/skills/`, `.codex/skills/` (all read) | `AGENTS.md` or `.cursor/rules/*.mdc` | `.cursor/hooks.json` | camelCase: `postToolUse` etc. | stdin JSON | JSON envelope (verify exact shape during impl) | +| Antigravity | `.agents/skills//SKILL.md` | `AGENTS.md` | undocumented at time of writing | unknown | unknown | unknown | Sources (saved for citation): - [Agent Skills open standard](https://agentskills.io) -- Claude Code: [skills](https://code.claude.com/docs/en/skills), [hooks](https://code.claude.com/docs/en/hooks) -- Codex: [skills](https://developers.openai.com/codex/skills/), [hooks](https://developers.openai.com/codex/hooks), [AGENTS.md guide](https://developers.openai.com/codex/guides/agents-md) -- OpenCode: [skills](https://opencode.ai/docs/skills/), [agents](https://opencode.ai/docs/agents/), [rules](https://opencode.ai/docs/rules/) -- Cursor: [skills](https://cursor.com/docs/context/skills), [rules](https://cursor.com/docs/rules), [hooks](https://cursor.com/docs/hooks) -- Antigravity: [authoring skills codelab](https://codelabs.developers.google.com/getting-started-with-antigravity-skills), [Gemini CLI configuration (inherited)](https://github.com/google-gemini/gemini-cli/tree/main/docs/hooks) +- Claude Code: [skills](https://code.claude.com/docs/en/skills), + [hooks](https://code.claude.com/docs/en/hooks) +- Codex: [skills](https://developers.openai.com/codex/skills/), + [hooks](https://developers.openai.com/codex/hooks), + [AGENTS.md guide](https://developers.openai.com/codex/guides/agents-md) +- OpenCode: [skills](https://opencode.ai/docs/skills/), + [agents](https://opencode.ai/docs/agents/), + [rules](https://opencode.ai/docs/rules/) +- Cursor: [skills](https://cursor.com/docs/context/skills), + [rules](https://cursor.com/docs/rules), [hooks](https://cursor.com/docs/hooks) +- Antigravity: + [authoring skills codelab](https://codelabs.developers.google.com/getting-started-with-antigravity-skills), + [Gemini CLI configuration (inherited)](https://github.com/google-gemini/gemini-cli/tree/main/docs/hooks) ## Implementation impact (sketch — for sizing only) -Updated for the post-pivot architecture. Pre-pivot bullets (shell-script bundling, jq dependency, no new mdvs subcommands) are obsolete — see the "Pivot to cross-platform mdvs-internal hooks" section above for what replaced them. - -- **New CLI subcommand `mdvs hook handle --platform --kind `** — the runtime that hooks call. Reads stdin JSON via serde, walks up to `mdvs.toml`, invokes `cmd::check` directly (no subprocess), wraps the markdown body in the per-platform envelope from `platform.toml`, emits to stdout, exits 0. -- **New CLI subcommand structure under `crates/mdvs/src/cmd/scaffold/`** (scaffold) and `crates/mdvs/src/cmd/hook/` (hook). The two share a `Platform` loader. +Updated for the post-pivot architecture. Pre-pivot bullets (shell-script +bundling, jq dependency, no new mdvs subcommands) are obsolete — see the "Pivot +to cross-platform mdvs-internal hooks" section above for what replaced them. + +- **New CLI subcommand + `mdvs hook handle --platform --kind `** — the + runtime that hooks call. Reads stdin JSON via serde, walks up to `mdvs.toml`, + invokes `cmd::check` directly (no subprocess), wraps the markdown body in the + per-platform envelope from `platform.toml`, emits to stdout, exits 0. +- **New CLI subcommand structure under `crates/mdvs/src/cmd/scaffold/`** + (scaffold) and `crates/mdvs/src/cmd/hook/` (hook). The two share a `Platform` + loader. - **Reshape `crates/mdvs/scaffolding/` to be config-only:** - Keep: `skill/SKILL.md`, `snippet/{agents-md.md,cursor-rules.mdc}`. - - Add: `platforms//platform.toml` for each supported harness (claude-code, codex, cursor, opencode, antigravity). - - Remove: `hooks//*.sh` and `{settings,hooks}.json` templates (the templates get *generated* by `mdvs scaffold hook` from `platform.toml` rather than bundled). -- **`Platform` struct + loader in `crates/mdvs/src/scaffold/platform.rs`** (new module). Plain struct, deserialised from `platform.toml` via `serde + toml`. No enum, no `dyn Trait` — the project's enum-dispatch rule still applies for backends/embedders/etc., but platforms are data-driven (new platform = new toml file, no Rust release). -- **Bundle `scaffolding/` into the binary** via `include_dir!` (or equivalent). Users get a single mdvs binary that contains all the per-platform data. -- **Skill body refresh** (`crates/mdvs/scaffolding/skill/SKILL.md`, currently 488 lines): drop the section that taught the agent to expect `jq`-wrapped envelopes (mdvs handles the wrapping now); the agent only needs to know about the `additionalContext` semantic. Replace `mdvs skill` references with `mdvs scaffold skill`. The schema-evolution loop content stays. -- **Hard rename `mdvs skill` → `mdvs scaffold skill`** (breaking, pre-1.0 so acceptable). -- **Update `book/src/recipes/agentic-harnesses-and-agentic-ides.md`** to use the new commands and the no-shell install path. Drop references to `validate.sh` / `search-nudge.sh` files; the recipe shows users dropping a one-line `command:` into their harness config. -- **Delete the shell scripts from commits `63311cf` + `01d8ff8`** as part of the cutover. The work isn't lost — it documented the executable spec that `mdvs hook handle` now implements in Rust. + - Add: `platforms//platform.toml` for each supported harness + (claude-code, codex, cursor, opencode, antigravity). + - Remove: `hooks//*.sh` and `{settings,hooks}.json` templates (the + templates get _generated_ by `mdvs scaffold hook` from `platform.toml` + rather than bundled). +- **`Platform` struct + loader in `crates/mdvs/src/scaffold/platform.rs`** (new + module). Plain struct, deserialised from `platform.toml` via `serde + toml`. + No enum, no `dyn Trait` — the project's enum-dispatch rule still applies for + backends/embedders/etc., but platforms are data-driven (new platform = new + toml file, no Rust release). +- **Bundle `scaffolding/` into the binary** via `include_dir!` (or equivalent). + Users get a single mdvs binary that contains all the per-platform data. +- **Skill body refresh** (`crates/mdvs/scaffolding/skill/SKILL.md`, currently + 488 lines): drop the section that taught the agent to expect `jq`-wrapped + envelopes (mdvs handles the wrapping now); the agent only needs to know about + the `additionalContext` semantic. Replace `mdvs skill` references with + `mdvs scaffold skill`. The schema-evolution loop content stays. +- **Hard rename `mdvs skill` → `mdvs scaffold skill`** (breaking, pre-1.0 so + acceptable). +- **Update `book/src/recipes/agentic-harnesses-and-agentic-ides.md`** to use the + new commands and the no-shell install path. Drop references to `validate.sh` / + `search-nudge.sh` files; the recipe shows users dropping a one-line `command:` + into their harness config. +- **Delete the shell scripts from commits `63311cf` + `01d8ff8`** as part of the + cutover. The work isn't lost — it documented the executable spec that + `mdvs hook handle` now implements in Rust. ## Implementation plan -Restructured after the 2026-06-22 pivot. Steps 1–3 of the original plan shipped in commits `63311cf` + `01d8ff8` and produced the shell-script scaffolding that's now being superseded — those steps are kept for history; new pivot-aware steps follow. +Restructured after the 2026-06-22 pivot. Steps 1–3 of the original plan shipped +in commits `63311cf` + `01d8ff8` and produced the shell-script scaffolding +that's now being superseded — those steps are kept for history; new pivot-aware +steps follow. ### Step 1 — Settle the load-bearing design decisions (≈ 30 min) ✓ DONE 2026-06-22 -All twelve open design questions settled. See "Decisions" section above. (Decisions #11 and #12 were later reversed by the pivot.) +All twelve open design questions settled. See "Decisions" section above. +(Decisions #11 and #12 were later reversed by the pivot.) ### Step 2 — Author all content artifacts (3–4 hours) ✓ DONE 2026-06-22 (commit `63311cf`) Produced under `crates/mdvs/scaffolding/`: + - `skill/SKILL.md` - `snippet/agents-md.md`, `snippet/cursor-rules.mdc` - `hooks/claude-code/validate.sh` + `settings.json` -End-to-end tested against `example_kb` on Claude Code. Caught one real bug (silent gate using output emptiness instead of exit code). +End-to-end tested against `example_kb` on Claude Code. Caught one real bug +(silent gate using output emptiness instead of exit code). ### Step 3 — Add Codex + Cursor hook variants (1–2 hours) ✓ DONE 2026-06-22 (commit `01d8ff8`) -Codex + Cursor `.sh` + config templates, plus extraction of the search-nudge to its own script (with cwd-based walk-up replacing the brittle `*kb/*` pattern), plus full `example_kb` dogfooding across `.claude` / `.agents` / `.codex` / `.cursor`. +Codex + Cursor `.sh` + config templates, plus extraction of the search-nudge to +its own script (with cwd-based walk-up replacing the brittle `*kb/*` pattern), +plus full `example_kb` dogfooding across `.claude` / `.agents` / `.codex` / +`.cursor`. ### Step 3b — Pivot decision (≈ 0 min, conversational) ✓ DONE 2026-06-22 -Recognised that the shell-script approach won't ship on Windows. Decision to pull hook logic into `mdvs hook handle` as a cross-platform Rust subcommand and make platform behaviour data-driven via `scaffolding/platforms//platform.toml`. See ["Pivot to cross-platform mdvs-internal hooks"](#pivot-to-cross-platform-mdvs-internal-hooks). +Recognised that the shell-script approach won't ship on Windows. Decision to +pull hook logic into `mdvs hook handle` as a cross-platform Rust subcommand and +make platform behaviour data-driven via +`scaffolding/platforms//platform.toml`. See +["Pivot to cross-platform mdvs-internal hooks"](#pivot-to-cross-platform-mdvs-internal-hooks). ### Step 4 — Design and write `platform.toml` for each supported harness (1–2 hours) -Write the per-platform config files that the new `mdvs hook handle` + `mdvs scaffold` commands read at runtime: +Write the per-platform config files that the new `mdvs hook handle` + +`mdvs scaffold` commands read at runtime: - `crates/mdvs/scaffolding/platforms/claude-code/platform.toml` - `crates/mdvs/scaffolding/platforms/codex/platform.toml` - `crates/mdvs/scaffolding/platforms/cursor/platform.toml` -- `crates/mdvs/scaffolding/platforms/opencode/platform.toml` (skill + snippet only, no `[hooks]` table) +- `crates/mdvs/scaffolding/platforms/opencode/platform.toml` (skill + snippet + only, no `[hooks]` table) - `crates/mdvs/scaffolding/platforms/antigravity/platform.toml` (same) -Schema sketch is in the Pivot section. Validate each one parses with `toml::from_str` round-tripping into the eventual `Platform` struct. +Schema sketch is in the Pivot section. Validate each one parses with +`toml::from_str` round-tripping into the eventual `Platform` struct. -**Verification:** Each toml file parses cleanly. The set of declared fields covers everything the shell scripts encoded as constants (event names, matchers, install paths, envelope shapes). +**Verification:** Each toml file parses cleanly. The set of declared fields +covers everything the shell scripts encoded as constants (event names, matchers, +install paths, envelope shapes). ### Step 5 — Implement `Platform` struct + loader in Rust (2–3 hours) New module `crates/mdvs/src/scaffold/platform.rs`: - `pub struct Platform { meta, skill, snippet, hooks: Option }` -- `Platform::load(name: &str) -> Result` — reads `scaffolding/platforms//platform.toml` (bundled via `include_dir!`) -- `Platform::list() -> Vec` — directory enumeration over bundled platforms -- `HooksConfig::emit_envelope(&self, msg, user_msg, kind) -> String` — templated substitution +- `Platform::load(name: &str) -> Result` — reads + `scaffolding/platforms//platform.toml` (bundled via `include_dir!`) +- `Platform::list() -> Vec` — directory enumeration over bundled + platforms +- `HooksConfig::emit_envelope(&self, msg, user_msg, kind) -> String` — templated + substitution -Add `include_dir` (or equivalent) as a dependency to embed `scaffolding/` at build time. +Add `include_dir` (or equivalent) as a dependency to embed `scaffolding/` at +build time. -**Verification:** Unit tests load every bundled platform.toml, exercise the envelope-emit path with a few values, assert the output matches expected shapes. +**Verification:** Unit tests load every bundled platform.toml, exercise the +envelope-emit path with a few values, assert the output matches expected shapes. ### Step 6 — Implement `mdvs hook handle --platform --kind ` (3–4 hours) -The runtime that hooks call. Replaces `validate.sh` + `search-nudge.sh` from Step 2/3. - -- New module `crates/mdvs/src/cmd/hook/mod.rs` with the `Hook` subcommand enum and `Handle` variant. -- `crates/mdvs/src/cmd/hook/handle.rs` — reads stdin JSON via `serde_json::from_reader(stdin())`, walks up from `cwd` (or current cwd if not in stdin) to find `mdvs.toml`, runs `cmd::check` directly (no subprocess), wraps the output via `Platform::emit_envelope`, prints to stdout, exits 0. -- The validate path also runs check twice (markdown for agent, pretty for user) and caps the user message at `MAX_USER_LINES=15` with `...` truncation — matches the shell-script behaviour. -- The search-nudge path checks the command-line pattern for grep/rg/find/etc. and only emits if cwd is in a vault. - -**Verification:** Unit tests for both kinds (validate, search-nudge) across all three hook-supporting platforms (claude-code, codex, cursor). Hand-install on `example_kb` for Claude Code by swapping the `.claude/settings.json` `command:` to `mdvs hook handle --platform claude-code --kind validate` — verify the existing test cycle (bogus status edit → violation in additionalContext + systemMessage) still works. +The runtime that hooks call. Replaces `validate.sh` + `search-nudge.sh` from +Step 2/3. + +- New module `crates/mdvs/src/cmd/hook/mod.rs` with the `Hook` subcommand enum + and `Handle` variant. +- `crates/mdvs/src/cmd/hook/handle.rs` — reads stdin JSON via + `serde_json::from_reader(stdin())`, walks up from `cwd` (or current cwd if not + in stdin) to find `mdvs.toml`, runs `cmd::check` directly (no subprocess), + wraps the output via `Platform::emit_envelope`, prints to stdout, exits 0. +- The validate path also runs check twice (markdown for agent, pretty for user) + and caps the user message at `MAX_USER_LINES=15` with `...` truncation — + matches the shell-script behaviour. +- The search-nudge path checks the command-line pattern for grep/rg/find/etc. + and only emits if cwd is in a vault. + +**Verification:** Unit tests for both kinds (validate, search-nudge) across all +three hook-supporting platforms (claude-code, codex, cursor). Hand-install on +`example_kb` for Claude Code by swapping the `.claude/settings.json` `command:` +to `mdvs hook handle --platform claude-code --kind validate` — verify the +existing test cycle (bogus status edit → violation in additionalContext + +systemMessage) still works. ### Step 7 — Implement `mdvs scaffold {skill,snippet,hook}` (2–3 hours) -The install-time generator commands. All three read `Platform` config to know what to emit. +The install-time generator commands. All three read `Platform` config to know +what to emit. -- `crates/mdvs/src/cmd/scaffold/mod.rs` with `Scaffold` subcommand enum (Skill, Snippet, Hook). -- `scaffold skill` — prints `scaffolding/skill/SKILL.md`. `--platform ` only changes the install-path comment in help text. -- `scaffold snippet` — prints `scaffolding/snippet/.md` where `` comes from `platform.toml`. For Cursor, also offers the `.mdc` wrap. -- `scaffold hook` — emits the platform-specific config (e.g. `.claude/settings.json` block) with the `command:` filled in as `mdvs hook handle --platform --kind `. No more `.sh` files emitted. +- `crates/mdvs/src/cmd/scaffold/mod.rs` with `Scaffold` subcommand enum (Skill, + Snippet, Hook). +- `scaffold skill` — prints `scaffolding/skill/SKILL.md`. `--platform ` + only changes the install-path comment in help text. +- `scaffold snippet` — prints `scaffolding/snippet/.md` where `` + comes from `platform.toml`. For Cursor, also offers the `.mdc` wrap. +- `scaffold hook` — emits the platform-specific config (e.g. + `.claude/settings.json` block) with the `command:` filled in as + `mdvs hook handle --platform --kind `. No more `.sh` files + emitted. -Remove `crates/mdvs/src/cmd/skill.rs` (hard rename, decision #6). Wire `scaffold` and `hook` into `main.rs`. +Remove `crates/mdvs/src/cmd/skill.rs` (hard rename, decision #6). Wire +`scaffold` and `hook` into `main.rs`. -**Verification:** `mdvs scaffold {skill,snippet,hook} --platform ` for each platform; output of `scaffold hook` parses as valid JSON. +**Verification:** `mdvs scaffold {skill,snippet,hook} --platform ` for +each platform; output of `scaffold hook` parses as valid JSON. ### Step 8 — Cutover `example_kb` to the no-shell install (≈ 30 min) -Delete the `.sh` symlinks under `example_kb/.{claude,codex,cursor}/hooks/`. Regenerate each platform's `settings.json` / `hooks.json` to call `mdvs hook handle` directly. Verify by re-running the bogus-status edit cycle (should produce identical agent+user output). +Delete the `.sh` symlinks under `example_kb/.{claude,codex,cursor}/hooks/`. +Regenerate each platform's `settings.json` / `hooks.json` to call +`mdvs hook handle` directly. Verify by re-running the bogus-status edit cycle +(should produce identical agent+user output). -**Verification:** Hooks fire the same way as before. No `jq` invocations show up in any installed config. Files removed: `.sh` symlinks; files modified: `settings.json` / `hooks.json` per platform. +**Verification:** Hooks fire the same way as before. No `jq` invocations show up +in any installed config. Files removed: `.sh` symlinks; files modified: +`settings.json` / `hooks.json` per platform. ### Step 9 — Delete the obsolete shell scripts from the scaffolding tree (≈ 15 min) -Remove `crates/mdvs/scaffolding/hooks//{validate,search-nudge}.sh` and `{settings,hooks}.json` (the templates — the new ones come from `platform.toml` + Rust code, not from bundled files). +Remove `crates/mdvs/scaffolding/hooks//{validate,search-nudge}.sh` and +`{settings,hooks}.json` (the templates — the new ones come from +`platform.toml` + Rust code, not from bundled files). `cargo build` clean. No references in code to the deleted paths. @@ -459,15 +724,20 @@ Remove `crates/mdvs/scaffolding/hooks//{validate,search-nudge}.sh` and Edit `crates/mdvs/scaffolding/skill/SKILL.md`: -- Drop the "hook output format" subsection that taught the agent about `jq` and the JSON envelope mechanics — that's all internal to mdvs now. -- Keep the schema-evolution warning loop (Step 4 of the skill) — the agent's *response* to violations is unchanged; only the delivery mechanism is. -- Trim references to `.claude/hooks/mdvs-validate.sh` etc. — the install no longer involves shell files. +- Drop the "hook output format" subsection that taught the agent about `jq` and + the JSON envelope mechanics — that's all internal to mdvs now. +- Keep the schema-evolution warning loop (Step 4 of the skill) — the agent's + _response_ to violations is unchanged; only the delivery mechanism is. +- Trim references to `.claude/hooks/mdvs-validate.sh` etc. — the install no + longer involves shell files. -**Verification:** Skill body shrinks slightly. Word "jq" no longer appears outside historical context. Agent still gets the warning-loop procedure intact. +**Verification:** Skill body shrinks slightly. Word "jq" no longer appears +outside historical context. Agent still gets the warning-loop procedure intact. ### Step 11 — Update the recipe page for the no-shell install (≈ 1 hour) -`book/src/recipes/agentic-harnesses-and-agentic-ides.md` — replace the hook section with the new shape: +`book/src/recipes/agentic-harnesses-and-agentic-ides.md` — replace the hook +section with the new shape: ```bash mdvs scaffold skill > .agents/skills/mdvs/SKILL.md @@ -475,13 +745,20 @@ mdvs scaffold snippet >> AGENTS.md mdvs scaffold hook --platform claude-code # prints settings.json snippet ``` -The output of `scaffold hook` is a single small JSON block users paste into their harness config. No script files, no `jq`, works on every OS. Note this in the "What's tested" section: Claude Code end-to-end; Codex / Cursor schema-correct but untested by us; Windows untested but architecturally supported (Rust binary, no shell). +The output of `scaffold hook` is a single small JSON block users paste into +their harness config. No script files, no `jq`, works on every OS. Note this in +the "What's tested" section: Claude Code end-to-end; Codex / Cursor +schema-correct but untested by us; Windows untested but architecturally +supported (Rust binary, no shell). -**Verification:** `mdbook build book` clean. Page is shorter than the current Unix-only version. +**Verification:** `mdbook build book` clean. Page is shorter than the current +Unix-only version. ### Step 12 — End-to-end test on Claude Code (the gate test, ≈ 1 hour) -Same shape as the original Step 9: bogus-status edit in `example_kb`, expect violation through `additionalContext` + `systemMessage`. Now exercising `mdvs hook handle` end-to-end. +Same shape as the original Step 9: bogus-status edit in `example_kb`, expect +violation through `additionalContext` + `systemMessage`. Now exercising +`mdvs hook handle` end-to-end. ### Step 13 — End-to-end test on Antigravity CLI (skill + snippet only, ≈ 30 min) @@ -490,27 +767,55 @@ Same as the original Step 10. ### Step 14 — PR and merge (≈ 30 min) - Open PR `feat/mdvs-wire` → `main`. -- CI must pass: `cargo test`, `cargo clippy --all-targets --features testing-mocks`, `mdbook build book`, `just lint-ast`. +- CI must pass: `cargo test`, + `cargo clippy --all-targets --features testing-mocks`, `mdbook build book`, + `just lint-ast`. - Self-review the diff. - Merge. - Subsume [TODO-0187](TODO-0187.md): mark `status: done`, `subsumed_by: 190`. -Post-merge: cocogitto auto-bumps. The rename of `mdvs skill` + the new `mdvs hook` subcommand are minor-version material (`0.7.x` → `0.8.0`). +Post-merge: cocogitto auto-bumps. The rename of `mdvs skill` + the new +`mdvs hook` subcommand are minor-version material (`0.7.x` → `0.8.0`). -**Total estimate post-pivot: 12–15 hours of focused work, spreading across 2–3 sessions.** (Roughly the same as the original plan — the Rust impl trades off against the shell-script ship that no longer needs polishing.) +**Total estimate post-pivot: 12–15 hours of focused work, spreading across 2–3 +sessions.** (Roughly the same as the original plan — the Rust impl trades off +against the shell-script ship that no longer needs polishing.) ## Verification (sketch) -Realistic test scope: we have direct access to **Claude Code and Antigravity CLI** for end-to-end verification on macOS. Codex, OpenCode, and Cursor will ship best-effort against their documented schemas, but without an actual end-to-end smoke test on each. Windows is architecturally supported (Rust binary, no shell dependency) but we haven't smoke-tested any harness on Windows yet. The recipe page should call this out honestly. `--help` output stays concise and doesn't mention test coverage. - -- All `mdvs scaffold {skill,snippet,hook} --platform ` invocations produce output that matches the documented schema for each platform (verifiable without running the harness — sufficient for the platforms we can't test live). -- `mdvs hook handle --platform --kind ` unit tests pass for every platform + kind combination on the CI matrix (Linux, macOS, Windows). -- **End-to-end test on Claude Code (macOS)** — full loop with all three artifacts wired up. Agent edits a markdown file in a vault with `mdvs.toml`, the validate-on-write hook fires, the agent receives the markdown explanation via `additionalContext` plus the truncated pretty render via `systemMessage`, and self-corrects on the next turn. Schema-evolution path: an intentional deviation results in the agent proposing an `mdvs.toml` update rather than silently fixing the file. -- **End-to-end test on Antigravity CLI (macOS)** — skill + snippet (hook out of scope per the upstream-docs gap). Confirms the cross-harness skill works in `.agents/skills/` and the AGENTS.md snippet is picked up. -- **No end-to-end test on Codex / OpenCode / Cursor (any OS), or any harness on Windows** — configs are documented-schema-correct but not smoke-tested by us. If a user reports a wiring bug for those, we treat it as a bug report rather than promising verified support. -- Recipe page renders cleanly and no longer contains hand-written hook JSON or shell-script content. +Realistic test scope: we have direct access to **Claude Code and Antigravity +CLI** for end-to-end verification on macOS. Codex, OpenCode, and Cursor will +ship best-effort against their documented schemas, but without an actual +end-to-end smoke test on each. Windows is architecturally supported (Rust +binary, no shell dependency) but we haven't smoke-tested any harness on Windows +yet. The recipe page should call this out honestly. `--help` output stays +concise and doesn't mention test coverage. + +- All `mdvs scaffold {skill,snippet,hook} --platform ` invocations produce + output that matches the documented schema for each platform (verifiable + without running the harness — sufficient for the platforms we can't test + live). +- `mdvs hook handle --platform --kind ` unit tests pass for every + platform + kind combination on the CI matrix (Linux, macOS, Windows). +- **End-to-end test on Claude Code (macOS)** — full loop with all three + artifacts wired up. Agent edits a markdown file in a vault with `mdvs.toml`, + the validate-on-write hook fires, the agent receives the markdown explanation + via `additionalContext` plus the truncated pretty render via `systemMessage`, + and self-corrects on the next turn. Schema-evolution path: an intentional + deviation results in the agent proposing an `mdvs.toml` update rather than + silently fixing the file. +- **End-to-end test on Antigravity CLI (macOS)** — skill + snippet (hook out of + scope per the upstream-docs gap). Confirms the cross-harness skill works in + `.agents/skills/` and the AGENTS.md snippet is picked up. +- **No end-to-end test on Codex / OpenCode / Cursor (any OS), or any harness on + Windows** — configs are documented-schema-correct but not smoke-tested by us. + If a user reports a wiring bug for those, we treat it as a bug report rather + than promising verified support. +- Recipe page renders cleanly and no longer contains hand-written hook JSON or + shell-script content. - `mdvs skill` is removed; help text points users to `mdvs scaffold skill`. -- No `.sh` files remain under `crates/mdvs/scaffolding/`; no `jq` references in any installed config that mdvs generates. +- No `.sh` files remain under `crates/mdvs/scaffolding/`; no `jq` references in + any installed config that mdvs generates. ## Out of scope @@ -518,5 +823,7 @@ Realistic test scope: we have direct access to **Claude Code and Antigravity CLI - `mdvs scaffold all` super-command (v2). - OpenCode TypeScript plugin emission (refuse with pointer in v1). - Antigravity hook emission (refuse with pointer in v1 — docs incomplete). -- Pre-explain (v2) and pre-validate (v3) hook modes — tracked in [TODO-0186](TODO-0186.md) and [TODO-0188](TODO-0188.md) respectively; will ship as follow-ups when those land. +- Pre-explain (v2) and pre-validate (v3) hook modes — tracked in + [TODO-0186](TODO-0186.md) and [TODO-0188](TODO-0188.md) respectively; will + ship as follow-ups when those land. - Real-time validation feedback inside the editor (LSP-style — separate scope). diff --git a/docs/spec/todos/TODO-0191.md b/docs/spec/todos/TODO-0191.md index 8cb08d2..820be40 100644 --- a/docs/spec/todos/TODO-0191.md +++ b/docs/spec/todos/TODO-0191.md @@ -25,15 +25,29 @@ files_updated: ## Summary -When a user (or an agent) writes `--where "tags = 'rust'"` against an `Array(String)` field, mdvs's `--where` translator passes the clause through as-is (with a `data.` prefix), and Lance/DataFusion rejects it because a `List(Utf8)` column can't be compared to a `Utf8` scalar with `=`. The user has to know to write `array_has(tags, 'rust')` instead — which is documented but easy to miss, and reads as a footgun. - -The right behavior: when an array field is compared with `=`, `!=`, `IN`, or `NOT IN` against a scalar literal of the element type, automatically rewrite to the equivalent `array_has(...)` form. The schema is already known at `--where`-translation time (mdvs reads `mdvs.toml` to discover `data_children` and `float_list_fields`), so the type information for "is this field an array?" is available. - -To keep the rewrite legible and avoid the "magic" feeling, the search outcome surfaces a translation note at the top showing each original → rewritten pair. +When a user (or an agent) writes `--where "tags = 'rust'"` against an +`Array(String)` field, mdvs's `--where` translator passes the clause through +as-is (with a `data.` prefix), and Lance/DataFusion rejects it because a +`List(Utf8)` column can't be compared to a `Utf8` scalar with `=`. The user has +to know to write `array_has(tags, 'rust')` instead — which is documented but +easy to miss, and reads as a footgun. + +The right behavior: when an array field is compared with `=`, `!=`, `IN`, or +`NOT IN` against a scalar literal of the element type, automatically rewrite to +the equivalent `array_has(...)` form. The schema is already known at +`--where`-translation time (mdvs reads `mdvs.toml` to discover `data_children` +and `float_list_fields`), so the type information for "is this field an array?" +is available. + +To keep the rewrite legible and avoid the "magic" feeling, the search outcome +surfaces a translation note at the top showing each original → rewritten pair. ## Context -Discovered 2026-06-23 when a Cursor agent invoked `mdvs search "..." --where "tags = 'rust'"` against the Refractions personal vault (matching the example in the bundled SKILL.md and `mdvs --help`'s `long_help`). The query failed at Lance: +Discovered 2026-06-23 when a Cursor agent invoked +`mdvs search "..." --where "tags = 'rust'"` against the Refractions personal +vault (matching the example in the bundled SKILL.md and `mdvs --help`'s +`long_help`). The query failed at Lance: ``` Error: lance error: Invalid user input: Error resolving filter expression @@ -41,51 +55,92 @@ data.tags = 'rust': Invalid user input: Received literal Utf8("rust") and could not convert to literal of type 'List(Field { data_type: Utf8, nullable: true })' ``` -A documentation patch landed in PR #66 (commit `a2f310d`) switching the examples to `array_has(tags, 'rust')`. That papered over the immediate symptom but doesn't fix the underlying ergonomics: users who guess the obvious form still hit the Lance error. A second hit confirmed the persistence: `--where "tags = 'smart-digital-twin'"` failed in exactly the same shape on 2026-06-24. +A documentation patch landed in PR #66 (commit `a2f310d`) switching the examples +to `array_has(tags, 'rust')`. That papered over the immediate symptom but +doesn't fix the underlying ergonomics: users who guess the obvious form still +hit the Lance error. A second hit confirmed the persistence: +`--where "tags = 'smart-digital-twin'"` failed in exactly the same shape on +2026-06-24. ## Design — parser-based translator -The current `translate_where_to_struct` (in `crates/mdvs/src/index/backend/search.rs`) is a regex pass: two regexes (`lit` for string literals, `ident` for identifiers), one walk-and-rewrite loop, no operator awareness. Growing it for `=` / `!=` / `IN` / `NOT IN` / "don't double-rewrite inside function calls" would mean adding more regexes, more peek-ahead heuristics, and more fragility every time a new rule lands. +The current `translate_where_to_struct` (in +`crates/mdvs/src/index/backend/search.rs`) is a regex pass: two regexes (`lit` +for string literals, `ident` for identifiers), one walk-and-rewrite loop, no +operator awareness. Growing it for `=` / `!=` / `IN` / `NOT IN` / "don't +double-rewrite inside function calls" would mean adding more regexes, more +peek-ahead heuristics, and more fragility every time a new rule lands. Switching to a real SQL parser is the right altitude for this work: -- **`sqlparser-rs` AST.** Returns `Expr::BinaryOp`, `Expr::InList`, `Expr::Identifier`, `Expr::Function`, etc. Pattern-matching on the AST is exact; "don't rewrite inside a function call" becomes free (structurally we know when we're inside `Expr::Function`). -- **Already in the dependency graph.** Pulled in transitively via datafusion → lance, so no new direct dependency tree weight — and using the same parser as Lance itself guarantees the rewrite output stays parseable downstream. -- **Future-proof.** Each future `--where` enhancement (LIKE on array elements? CONTAINS-ALL? regex?) becomes one match arm in the AST walk rather than another regex. +- **`sqlparser-rs` AST.** Returns `Expr::BinaryOp`, `Expr::InList`, + `Expr::Identifier`, `Expr::Function`, etc. Pattern-matching on the AST is + exact; "don't rewrite inside a function call" becomes free (structurally we + know when we're inside `Expr::Function`). +- **Already in the dependency graph.** Pulled in transitively via datafusion → + lance, so no new direct dependency tree weight — and using the same parser as + Lance itself guarantees the rewrite output stays parseable downstream. +- **Future-proof.** Each future `--where` enhancement (LIKE on array elements? + CONTAINS-ALL? regex?) becomes one match arm in the AST walk rather than + another regex. Migration shape: -1. **Add `sqlparser` as a direct dependency** (alongside `lance` / `lancedb`). Pin to whatever version Lance is using to avoid version skew. -2. **Replace `translate_where_to_struct`** with a `parse → walk → emit` pipeline: +1. **Add `sqlparser` as a direct dependency** (alongside `lance` / `lancedb`). + Pin to whatever version Lance is using to avoid version skew. +2. **Replace `translate_where_to_struct`** with a `parse → walk → emit` + pipeline: - Parse the user's clause as a DataFusion-dialect SQL expression. - Walk the `Expr` tree. At each node, apply: - - **Identifier rewrite** (unchanged from today): if the identifier names a `data` child, qualify it with `data.`; if it names an internal column, resolve via aliases/prefix; if it names an `Array(Float)` field, error out. + - **Identifier rewrite** (unchanged from today): if the identifier names a + `data` child, qualify it with `data.`; if it names an internal column, + resolve via aliases/prefix; if it names an `Array(Float)` field, error + out. - **Array-equality rewrite** (NEW — see "Rewrite rules" below). - Re-emit the transformed AST as SQL via `Expr::to_string()`. -3. **Collect rewrites** during the walk as `Vec` and return them alongside the rewritten string. The caller (`search.rs`) threads them into the search outcome. +3. **Collect rewrites** during the walk as `Vec` and return them + alongside the rewritten string. The caller (`search.rs`) threads them into + the search outcome. ### Rewrite rules (4 forms) -For each of the rules below, both literal orderings are supported: ` OP ` and ` OP `. +For each of the rules below, both literal orderings are supported: +` OP ` and ` OP `. -| User wrote | Rewritten to | -|---|---| -| `tags = 'x'` | `array_has(data.tags, 'x')` | -| `tags != 'x'` | `NOT array_has(data.tags, 'x')` | -| `tags IN ('x', 'y')` | `array_has(data.tags, 'x') OR array_has(data.tags, 'y')` | +| User wrote | Rewritten to | +| ------------------------ | -------------------------------------------------------------- | +| `tags = 'x'` | `array_has(data.tags, 'x')` | +| `tags != 'x'` | `NOT array_has(data.tags, 'x')` | +| `tags IN ('x', 'y')` | `array_has(data.tags, 'x') OR array_has(data.tags, 'y')` | | `tags NOT IN ('x', 'y')` | `NOT (array_has(data.tags, 'x') OR array_has(data.tags, 'y'))` | -**Element semantics confirmed** for `!=` and `NOT IN`: both mean "the array does not contain this element". The alternative interpretation ("the array does not equal the singleton") is what 1% of users mean — and Lance can't express it anyway without `array_length` + `array_has` clauses nobody writes. Element semantics is what the user expects, gets documented as the contract. +**Element semantics confirmed** for `!=` and `NOT IN`: both mean "the array does +not contain this element". The alternative interpretation ("the array does not +equal the singleton") is what 1% of users mean — and Lance can't express it +anyway without `array_length` + `array_has` clauses nobody writes. Element +semantics is what the user expects, gets documented as the contract. -**Function-call invariants**: if the array field already appears inside a function (`array_has(tags, 'x')`, `array_length(tags) > 2`, `cast(tags as text)`), leave it alone. The AST makes this trivial — we only fire the rewrite on `Expr::BinaryOp` / `Expr::InList` where the operand is a bare `Expr::Identifier`, never on identifiers nested inside `Expr::Function`. +**Function-call invariants**: if the array field already appears inside a +function (`array_has(tags, 'x')`, `array_length(tags) > 2`, +`cast(tags as text)`), leave it alone. The AST makes this trivial — we only fire +the rewrite on `Expr::BinaryOp` / `Expr::InList` where the operand is a bare +`Expr::Identifier`, never on identifiers nested inside `Expr::Function`. -**Element types**: rewrite applies to `Array(String)`, `Array(Integer)`, `Array(Boolean)`, `Array(Date)`, `Array(DateTime)`. `Array(Float)` stays rejected up-front (TODO-0159, distinct bug). +**Element types**: rewrite applies to `Array(String)`, `Array(Integer)`, +`Array(Boolean)`, `Array(Date)`, `Array(DateTime)`. `Array(Float)` stays +rejected up-front (TODO-0159, distinct bug). ### Translation note -Without surfacing the rewrite, users hit results that look like they came from a clause they didn't write. The note explains it concisely at the top of the search output. +Without surfacing the rewrite, users hit results that look like they came from a +clause they didn't write. The note explains it concisely at the top of the +search output. -Add `where_rewrites: Vec` to `SearchOutcome`. When non-empty, render at the top as a `Block::Section` (markdown: `## Note — rewrote N array-field expressions`; pretty: italic block above the result table). Each entry shows `original → rewritten`. JSON output carries it as structured data. +Add `where_rewrites: Vec` to `SearchOutcome`. When non-empty, +render at the top as a `Block::Section` (markdown: +`## Note — rewrote N array-field expressions`; pretty: italic block above the +result table). Each entry shows `original → rewritten`. JSON output carries it +as structured data. Example pretty rendering: @@ -97,30 +152,50 @@ Searched "smart digital twin actor model" — 3 hits ... ``` -The translator returns the rewrites in original-clause column order; the renderer doesn't need to know anything about array semantics, just how to format the pairs. +The translator returns the rewrites in original-clause column order; the +renderer doesn't need to know anything about array semantics, just how to format +the pairs. ## Out of scope -- Auto-rewriting for `Array(Float)` (still rejected up front; the Lance-encoding bug from TODO-0159 isn't fixed by changing the translator). -- Auto-rewriting `LIKE` / `GLOB` / regex operators against array fields (would need different semantics — "any element matches the pattern" — and isn't requested yet). -- Schema-aware rewrites for dotted-name leaves (already work because dotted names address scalar leaves, not arrays). -- Opt-in / opt-out flag: the current behavior (fail with a Lance error) is strictly worse for anyone who doesn't already know the `array_has` incantation. The rewrite is always-on. +- Auto-rewriting for `Array(Float)` (still rejected up front; the Lance-encoding + bug from TODO-0159 isn't fixed by changing the translator). +- Auto-rewriting `LIKE` / `GLOB` / regex operators against array fields (would + need different semantics — "any element matches the pattern" — and isn't + requested yet). +- Schema-aware rewrites for dotted-name leaves (already work because dotted + names address scalar leaves, not arrays). +- Opt-in / opt-out flag: the current behavior (fail with a Lance error) is + strictly worse for anyone who doesn't already know the `array_has` + incantation. The rewrite is always-on. ## Verification -- Existing translator tests in `crates/mdvs/src/index/backend/search.rs::tests` all still pass (regression: the parser-based translator emits the same SQL for non-array clauses). -- New tests covering each rule (= / != / IN / NOT IN) for both operand orderings, both with and without `data.` prefix: +- Existing translator tests in `crates/mdvs/src/index/backend/search.rs::tests` + all still pass (regression: the parser-based translator emits the same SQL for + non-array clauses). +- New tests covering each rule (= / != / IN / NOT IN) for both operand + orderings, both with and without `data.` prefix: - `array_equality_rewrites_to_array_has` - `array_equality_literal_on_left` - `array_inequality_rewrites_to_not_array_has` - `array_in_list_rewrites_to_or_chain` - `array_not_in_list_rewrites_to_not_or_chain` - - `array_inside_function_call_not_double_rewritten` — `array_has(tags, 'x')` stays unchanged - - `array_inside_array_length_not_rewritten` — `array_length(tags) > 2` stays unchanged -- New `SearchOutcome::where_rewrites` populated in tests; render assertions for pretty + markdown + JSON. -- Doc updates (SKILL.md, README, `--help`) switch examples back to `tags = 'x'` as the natural form. -- End-to-end on `example_kb`: `mdvs search "" --where "tags = 'foo'"` returns results AND prints the translation note. + - `array_inside_function_call_not_double_rewritten` — `array_has(tags, 'x')` + stays unchanged + - `array_inside_array_length_not_rewritten` — `array_length(tags) > 2` stays + unchanged +- New `SearchOutcome::where_rewrites` populated in tests; render assertions for + pretty + markdown + JSON. +- Doc updates (SKILL.md, README, `--help`) switch examples back to `tags = 'x'` + as the natural form. +- End-to-end on `example_kb`: `mdvs search "" --where "tags = 'foo'"` returns + results AND prints the translation note. ## Impact -Removes a footgun. The previous "fix" was documentation-only — users who follow the docs work, but users who guess the obvious syntax hit a Lance-level error message that doesn't suggest the right form. Auto-rewrite means the obvious syntax just works; the translation note keeps the rewrite legible instead of magic. +Removes a footgun. The previous "fix" was documentation-only — users who follow +the docs work, but users who guess the obvious syntax hit a Lance-level error +message that doesn't suggest the right form. Auto-rewrite means the obvious +syntax just works; the translation note keeps the rewrite legible instead of +magic. diff --git a/docs/spec/todos/TODO-0192.md b/docs/spec/todos/TODO-0192.md index 30915e4..dbee620 100644 --- a/docs/spec/todos/TODO-0192.md +++ b/docs/spec/todos/TODO-0192.md @@ -16,11 +16,21 @@ files_updated: ## Summary -When a `cfg(test)`- or `--features testing-mocks`-enabled `mdvs` binary runs `build` (including the auto-build chain reached from `search`) against a vault whose `mdvs.toml` has no `[embedding_model]` block, `mutate_config` synthesizes a `provider = "mock"` default **and writes it to disk**. A subsequent run with the production binary (compiled without `testing-mocks`) reads that file and refuses with `embedding provider 'mock' is only available in builds compiled with --features testing-mocks`. The test-mode binary leaked its config into a real user vault. +When a `cfg(test)`- or `--features testing-mocks`-enabled `mdvs` binary runs +`build` (including the auto-build chain reached from `search`) against a vault +whose `mdvs.toml` has no `[embedding_model]` block, `mutate_config` synthesizes +a `provider = "mock"` default **and writes it to disk**. A subsequent run with +the production binary (compiled without `testing-mocks`) reads that file and +refuses with +`embedding provider 'mock' is only available in builds compiled with --features testing-mocks`. +The test-mode binary leaked its config into a real user vault. ## Context -Discovered 2026-06-23 against a personal vault. The user's `/usr/local/bin/mdvs` was a symlink to a `cargo build` debug binary (`cfg(test)` enabled the mock default). At some point during the session it ran `mdvs search` → auto-build → `mutate_config`, which saw `embedding_model = None` and wrote: +Discovered 2026-06-23 against a personal vault. The user's `/usr/local/bin/mdvs` +was a symlink to a `cargo build` debug binary (`cfg(test)` enabled the mock +default). At some point during the session it ran `mdvs search` → auto-build → +`mutate_config`, which saw `embedding_model = None` and wrote: ```toml [embedding_model] @@ -29,29 +39,58 @@ name = "mock" dim = 256 ``` -After that, the production binary on PATH (installed via `cargo install`) reads `mdvs.toml`, sees `provider = "mock"`, and bails at `EmbedderConfig::from_config` with the "only available in builds compiled with `--features testing-mocks`" error from `crates/mdvs/src/index/embed.rs:40`. +After that, the production binary on PATH (installed via `cargo install`) reads +`mdvs.toml`, sees `provider = "mock"`, and bails at +`EmbedderConfig::from_config` with the "only available in builds compiled with +`--features testing-mocks`" error from `crates/mdvs/src/index/embed.rs:40`. -The intent of the cfg-gated mock default (TODO-0184) is correct: tests that flow through `init → build` shouldn't reach for HuggingFace. But the persistence is wrong — a test-mode config should never escape to the user's committed `mdvs.toml`. +The intent of the cfg-gated mock default (TODO-0184) is correct: tests that flow +through `init → build` shouldn't reach for HuggingFace. But the persistence is +wrong — a test-mode config should never escape to the user's committed +`mdvs.toml`. ## Resolution -In `crates/mdvs/src/cmd/build/config_mutate.rs::mutate_config`, split the `None` arm of the `match config.embedding_model` into two `cfg`-gated branches and remove `config_changed = true` from the mock branch: - -- Production (`cfg(not(any(test, feature = "testing-mocks")))`): synthesize the `model2vec` default AND set `config_changed = true`, so first-time `build` persists the real `[embedding_model]` block. -- Test/mock (`cfg(any(test, feature = "testing-mocks"))`): synthesize the mock default in-memory only. `config_changed` stays false on this branch, so `mdvs.toml` is never rewritten. The mock default exists only for the duration of the run. - -The `Some(...)` arms (existing `[embedding_model]` block, with or without `--set-model`/`--set-revision`) are unchanged — they still persist user-driven mutations. - -A regression test (`mock_embedder_default_is_not_persisted_to_mdvs_toml`) writes a minimal `mdvs.toml` without an `[embedding_model]` block, runs `mutate_config`, re-reads the file from disk, and asserts that neither `mock` nor `[embedding_model]` appears in the written file. The in-memory `config.embedding_model` is still verified to be the mock default, so the rest of the build chain still has a valid embedder to work with. Under `cfg(test)` this test exercises the mock branch directly. - -Existing tests (`second_build_skips_write_index_when_nothing_changed`, `third_build_persists_new_file_via_incremental_path`, etc.) continue to pass — they consume the in-memory `MdvsToml` after `build` and never depended on the mock provider being persisted to disk. +In `crates/mdvs/src/cmd/build/config_mutate.rs::mutate_config`, split the `None` +arm of the `match config.embedding_model` into two `cfg`-gated branches and +remove `config_changed = true` from the mock branch: + +- Production (`cfg(not(any(test, feature = "testing-mocks")))`): synthesize the + `model2vec` default AND set `config_changed = true`, so first-time `build` + persists the real `[embedding_model]` block. +- Test/mock (`cfg(any(test, feature = "testing-mocks"))`): synthesize the mock + default in-memory only. `config_changed` stays false on this branch, so + `mdvs.toml` is never rewritten. The mock default exists only for the duration + of the run. + +The `Some(...)` arms (existing `[embedding_model]` block, with or without +`--set-model`/`--set-revision`) are unchanged — they still persist user-driven +mutations. + +A regression test (`mock_embedder_default_is_not_persisted_to_mdvs_toml`) writes +a minimal `mdvs.toml` without an `[embedding_model]` block, runs +`mutate_config`, re-reads the file from disk, and asserts that neither `mock` +nor `[embedding_model]` appears in the written file. The in-memory +`config.embedding_model` is still verified to be the mock default, so the rest +of the build chain still has a valid embedder to work with. Under `cfg(test)` +this test exercises the mock branch directly. + +Existing tests (`second_build_skips_write_index_when_nothing_changed`, +`third_build_persists_new_file_via_incremental_path`, etc.) continue to pass — +they consume the in-memory `MdvsToml` after `build` and never depended on the +mock provider being persisted to disk. ## Out of scope - Changing the production default (still `model2vec` / `potion-base-8M`). -- Removing the `cfg`-gated mock default entirely. The TODO-0184 design (deterministic hermetic test embedder) stays. -- Auditing other call sites that may persist test-only defaults. None are known today; if more surface, file follow-ups. +- Removing the `cfg`-gated mock default entirely. The TODO-0184 design + (deterministic hermetic test embedder) stays. +- Auditing other call sites that may persist test-only defaults. None are known + today; if more surface, file follow-ups. ## Impact -A test-mode binary running against a user vault stops corrupting that vault's `mdvs.toml`. Without this fix, a developer who happens to symlink `target/debug/mdvs` into PATH and then runs a quick `mdvs search` against any real corpus permanently breaks that corpus for the production binary. +A test-mode binary running against a user vault stops corrupting that vault's +`mdvs.toml`. Without this fix, a developer who happens to symlink +`target/debug/mdvs` into PATH and then runs a quick `mdvs search` against any +real corpus permanently breaks that corpus for the production binary. diff --git a/docs/spec/todos/TODO-0193.md b/docs/spec/todos/TODO-0193.md index 5a274f1..8a20168 100644 --- a/docs/spec/todos/TODO-0193.md +++ b/docs/spec/todos/TODO-0193.md @@ -11,51 +11,108 @@ blocks: [] # TODO-0193: Support `.mdx` files — free validation, gated search-body stripping ## Summary -Teach mdvs to ingest `.mdx` files. MDX frontmatter is byte-identical to Markdown frontmatter, so the validation layer (`init` / `check` / `update`) works the moment the scanner accepts the extension — one line, zero new logic. The search layer needs more: MDX bodies carry `import`/`export` statements and `{expr}` JavaScript expressions that leak into embeddings as noise, so a fence-aware pre-chunk strip stage is required before `.mdx` bodies produce good search results. Ship the two halves as separate increments; the validation half unlocks docs-site frontmatter linting (MDX's actual unmet pain point) on its own. + +Teach mdvs to ingest `.mdx` files. MDX frontmatter is byte-identical to Markdown +frontmatter, so the validation layer (`init` / `check` / `update`) works the +moment the scanner accepts the extension — one line, zero new logic. The search +layer needs more: MDX bodies carry `import`/`export` statements and `{expr}` +JavaScript expressions that leak into embeddings as noise, so a fence-aware +pre-chunk strip stage is required before `.mdx` bodies produce good search +results. Ship the two halves as separate increments; the validation half unlocks +docs-site frontmatter linting (MDX's actual unmet pain point) on its own. ## Motivation -MDX (Markdown + JSX) is the dominant content format for docs-site pipelines — Docusaurus, Astro, Nextra, Contentlayer — where frontmatter *drives* the build (routing, sidebar order, tags). Nothing in that ecosystem validates frontmatter against a schema before the build breaks. There is no "Obsidian for MDX" either; MDX note-taking is done via Obsidian plugins that render `.mdx` as `.md`. mdvs's validation layer maps onto this niche almost for free. See the design conversation that produced this TODO for the landscape survey. + +MDX (Markdown + JSX) is the dominant content format for docs-site pipelines — +Docusaurus, Astro, Nextra, Contentlayer — where frontmatter _drives_ the build +(routing, sidebar order, tags). Nothing in that ecosystem validates frontmatter +against a schema before the build breaks. There is no "Obsidian for MDX" either; +MDX note-taking is done via Obsidian plugins that render `.mdx` as `.md`. mdvs's +validation layer maps onto this niche almost for free. See the design +conversation that produced this TODO for the landscape survey. ## Current behavior (grounding) -- **Ingest gate.** The default scan glob is `**` (`../schema/config.rs` default) — it matches every path — so the extension filter in `discover/scan.rs` is the *sole* markdown-only gate: + +- **Ingest gate.** The default scan glob is `**` (`../schema/config.rs` default) + — it matches every path — so the extension filter in `discover/scan.rs` is the + _sole_ markdown-only gate: ```rust .filter(|e| e.path().extension().is_some_and(|ext| ext == "md" || ext == "markdown")) ``` Everything downstream is extension-agnostic. -- **Frontmatter.** Engine selection keys off the leading delimiter (`detect_engine(&raw)` probing `---` / `+++` / `{`), never the extension. An MDX frontmatter block is byte-identical to a Markdown one → validation is unaffected by the body. -- **Body → search.** `index/chunk.rs::extract_plain_text` collects only `Event::Text` from pulldown-cmark. JSX *tag markup* (``) parses as `Event::Html` / `Event::InlineHtml` and is already dropped for free; text *between* tags (`Important` → `Important`) survives as `Event::Text`, which is desirable. +- **Frontmatter.** Engine selection keys off the leading delimiter + (`detect_engine(&raw)` probing `---` / `+++` / `{`), never the extension. An + MDX frontmatter block is byte-identical to a Markdown one → validation is + unaffected by the body. +- **Body → search.** `index/chunk.rs::extract_plain_text` collects only + `Event::Text` from pulldown-cmark. JSX _tag markup_ (``) + parses as `Event::Html` / `Event::InlineHtml` and is already dropped for free; + text _between_ tags (`Important` → `Important`) survives as + `Event::Text`, which is desirable. ## What leaks into embeddings (the search problem) + Parsed as `Event::Text` by a CommonMark parser, therefore embedded as noise: -1. **`import` / `export` statements** — every MDX file opens with a block of these; they are plain paragraphs to pulldown-cmark, not HTML. -2. **Expression braces** — `{frontmatter.title}`, `{2026 - startYear}`, `{items.map(...)}`. -3. **Splitter confusion** — JSX inside a paragraph can blur where `MarkdownSplitter` (`index/chunk.rs`) places semantic boundaries (quality wobble, not garbage). + +1. **`import` / `export` statements** — every MDX file opens with a block of + these; they are plain paragraphs to pulldown-cmark, not HTML. +2. **Expression braces** — `{frontmatter.title}`, `{2026 - startYear}`, + `{items.map(...)}`. +3. **Splitter confusion** — JSX inside a paragraph can blur where + `MarkdownSplitter` (`index/chunk.rs`) places semantic boundaries (quality + wobble, not garbage). ## Proposed work ### Increment A — ingest + validation (trivial, ship first) + - Add `|| ext == "mdx"` to the extension filter in `discover/scan.rs`. -- Add `.mdx` fixtures to the scan tests; confirm frontmatter detection + `check` behave identically to `.md`. -- No search changes. `build` / `search` will "work" on `.mdx` but with the noise described above — acceptable for an initial validation-focused release, documented as a known limitation. +- Add `.mdx` fixtures to the scan tests; confirm frontmatter detection + `check` + behave identically to `.md`. +- No search changes. `build` / `search` will "work" on `.mdx` but with the noise + described above — acceptable for an initial validation-focused release, + documented as a known limitation. ### Increment B — search-body strip (the real work, gated) -- New **fence-aware** pre-chunk strip stage, wired upstream of `Chunks::new` in `index/chunk.rs` (mirrors how `strip_wikilinks` runs as a targeted pass inside `extract_plain_text`). It must: + +- New **fence-aware** pre-chunk strip stage, wired upstream of `Chunks::new` in + `index/chunk.rs` (mirrors how `strip_wikilinks` runs as a targeted pass inside + `extract_plain_text`). It must: - remove leading/inline `import` / `export` lines, - remove `{expr}` expressions **outside** fenced code blocks, - - **never** touch content inside code fences — the `plain_text_preserves_code_block_content` test in `chunk.rs` guards this and it is desirable content. -- Because a fence-aware `{…}` strip is not a trivial regex (expressions nest and span lines; `{` appears in prose and code), this is the one genuine unit of implementation work. + - **never** touch content inside code fences — the + `plain_text_preserves_code_block_content` test in `chunk.rs` guards this and + it is desirable content. +- Because a fence-aware `{…}` strip is not a trivial regex (expressions nest and + span lines; `{` appears in prose and code), this is the one genuine unit of + implementation work. ## Design decisions to settle before coding -1. **Opt-in vs automatic strip.** Stripping `{…}` from a plain-Markdown `.md` file could eat legitimate prose. Recommend gating the strip by extension (`.mdx` gets stripped, `.md` does not) rather than a global transform; optionally back it with an explicit `[scan]` flag. Pure-Markdown vaults must not pay MDX cost or risk false strips. -2. **Strip aggressiveness.** Minimal (drop `import`/`export` lines + inline `{…}` outside fences) removes ~90% of the noise cheaply. Full JSX-tree removal is more correct but wants a real MDX AST (e.g. `markdown-rs` MDX mode) — heavier, likely not worth it for v0. Recommend minimal. -3. **Config surface.** If gated by a flag, decide the `[scan]` field name and default. If gated purely by extension, no config change. + +1. **Opt-in vs automatic strip.** Stripping `{…}` from a plain-Markdown `.md` + file could eat legitimate prose. Recommend gating the strip by extension + (`.mdx` gets stripped, `.md` does not) rather than a global transform; + optionally back it with an explicit `[scan]` flag. Pure-Markdown vaults must + not pay MDX cost or risk false strips. +2. **Strip aggressiveness.** Minimal (drop `import`/`export` lines + inline + `{…}` outside fences) removes ~90% of the noise cheaply. Full JSX-tree + removal is more correct but wants a real MDX AST (e.g. `markdown-rs` MDX + mode) — heavier, likely not worth it for v0. Recommend minimal. +3. **Config surface.** If gated by a flag, decide the `[scan]` field name and + default. If gated purely by extension, no config change. ## Non-goals -- No JSX/React rendering, component resolution, or evaluation — mdvs indexes prose, it does not build a site. -- No new heavy MDX-AST dependency in v0 unless decision (2) chooses full-tree removal. + +- No JSX/React rendering, component resolution, or evaluation — mdvs indexes + prose, it does not build a site. +- No new heavy MDX-AST dependency in v0 unless decision (2) chooses full-tree + removal. ## Files likely touched + - `crates/mdvs/src/discover/scan.rs` — extension gate (A) + tests. - `crates/mdvs/src/index/chunk.rs` — strip stage + tests (B). -- `crates/mdvs/src/schema/shared.rs` / `config.rs` — only if a `[scan]` flag is chosen (B). -- Spec: `../architecture.md` (pipeline note), possibly a `book/` page on MDX support. +- `crates/mdvs/src/schema/shared.rs` / `config.rs` — only if a `[scan]` flag is + chosen (B). +- Spec: `../architecture.md` (pipeline note), possibly a `book/` page on MDX + support. diff --git a/docs/spec/todos/TODO-0194.md b/docs/spec/todos/TODO-0194.md index a3935d2..d02ac79 100644 --- a/docs/spec/todos/TODO-0194.md +++ b/docs/spec/todos/TODO-0194.md @@ -13,7 +13,11 @@ related: [195] ## Summary -A frontmatter field that appears in no `[[fields.field]]` entry is never a violation — `check` reports it as informational "new fields" and exits 0, in both auto-update and `--no-update` modes. Add a directory-scoped policy section to `mdvs.toml` that can close that world: under a given path glob, undeclared fields become violations. +A frontmatter field that appears in no `[[fields.field]]` entry is never a +violation — `check` reports it as informational "new fields" and exits 0, in +both auto-update and `--no-update` modes. Add a directory-scoped policy section +to `mdvs.toml` that can close that world: under a given path glob, undeclared +fields become violations. ## Details @@ -35,26 +39,43 @@ EXIT: 0 Two mechanisms produce this: -- `cmd/check/validate.rs` iterates `[[fields.field]]` entries and calls `navigate_dotted` to find each declared field's value. It never iterates the *keys present in the file*, so a key mdvs has not been told about is never examined. -- `schema/json_schema/to_canonical.rs` emits the root schema with `additionalProperties: true`. `cmd/check/field_meta.rs` notes that per-field validation "would never produce `Required` or `AdditionalProperties` errors anyway." - -`ViolationKind::Disallowed` exists but means something narrower: a *declared* field appearing at a path outside its `allowed` globs (`validate.rs`, the `!m.allowed.is_match(file_path_str)` branch). - -What auto-update does and does not do, also measured — re-inference only ever **adds new fields**. It does not widen an existing field's type or relax its constraints: - -| Scenario | auto-update (default) | `--no-update` | -|---|---|---| -| Brand-new undeclared field | silently written into `mdvs.toml`, exit 0 | listed under "New fields", exit 0 | -| Value violating existing `categories` | exit 1 | exit 1 | -| Wrong type on existing field | exit 1 | exit 1 | - -So `--no-update` is not a gate against schema drift; it only prevents `mdvs.toml` from being rewritten mid-run and surfaces the addition in the output. +- `cmd/check/validate.rs` iterates `[[fields.field]]` entries and calls + `navigate_dotted` to find each declared field's value. It never iterates the + _keys present in the file_, so a key mdvs has not been told about is never + examined. +- `schema/json_schema/to_canonical.rs` emits the root schema with + `additionalProperties: true`. `cmd/check/field_meta.rs` notes that per-field + validation "would never produce `Required` or `AdditionalProperties` errors + anyway." + +`ViolationKind::Disallowed` exists but means something narrower: a _declared_ +field appearing at a path outside its `allowed` globs (`validate.rs`, the +`!m.allowed.is_match(file_path_str)` branch). + +What auto-update does and does not do, also measured — re-inference only ever +**adds new fields**. It does not widen an existing field's type or relax its +constraints: + +| Scenario | auto-update (default) | `--no-update` | +| ------------------------------------- | ----------------------------------------- | --------------------------------- | +| Brand-new undeclared field | silently written into `mdvs.toml`, exit 0 | listed under "New fields", exit 0 | +| Value violating existing `categories` | exit 1 | exit 1 | +| Wrong type on existing field | exit 1 | exit 1 | + +So `--no-update` is not a gate against schema drift; it only prevents +`mdvs.toml` from being rewritten mid-run and surfaces the addition in the +output. ### Why per-directory rather than global -mdvs already has a directory axis, but only *per field*: `allowed` and `required` are path globs on each `[[fields.field]]`. What is missing is the complementary **closed-world statement** — "under `notes/**`, only these fields may appear." Today `[fields].ignore` is global and the open/closed decision is all-or-nothing. +mdvs already has a directory axis, but only _per field_: `allowed` and +`required` are path globs on each `[[fields.field]]`. What is missing is the +complementary **closed-world statement** — "under `notes/**`, only these fields +may appear." Today `[fields].ignore` is global and the open/closed decision is +all-or-nothing. -A realistic vault wants both at once: a curated directory that is frozen, and a scratch directory where anything goes. A global switch cannot express that. +A realistic vault wants both at once: a curated directory that is frozen, and a +scratch directory where anything goes. A global switch cannot express that. ### Proposed shape @@ -71,34 +92,58 @@ path = "inbox/**" frozen = false # explicit open world ``` -`path = "**"` covers the whole-repo freeze, so the global case falls out of the same mechanism rather than needing its own flag. +`path = "**"` covers the whole-repo freeze, so the global case falls out of the +same mechanism rather than needing its own flag. ### Design questions -- **Precedence.** When a file matches several scopes, does the most specific glob win, or must every matching scope pass? Most-specific-wins matches how people read directory config; all-must-pass is easier to reason about formally. Needs a decision before implementation. -- **Relationship to `[fields].ignore`.** Does the global list stay and compose with per-scope `ignore`, or does it become sugar for a `path = "**"` scope? Prefer not having two mechanisms for the same thing. -- **Default for unmatched paths.** A file matching no scope should presumably stay open-world, preserving today's behavior for every existing vault. Confirm this is the migration story. -- **Which violation kind.** Reuse `Disallowed`, or add a distinct kind (`Undeclared`?) so the two cases stay legible in the output? `Disallowed` currently carries the rule string `allowed in [...]`, which would read oddly for this case. -- **Interaction with auto-update.** Inside a frozen scope, should `check`'s auto-update refuse to absorb a new field, rather than adding it and then flagging it? Probably yes, or the two features fight. -- **Does `frozen` belong per-scope or is it the only policy?** If `[[scope]]` is going to exist, consider what else is directory-scoped (see TODO-0195 for cross-field rules, which want a similar home). +- **Precedence.** When a file matches several scopes, does the most specific + glob win, or must every matching scope pass? Most-specific-wins matches how + people read directory config; all-must-pass is easier to reason about + formally. Needs a decision before implementation. +- **Relationship to `[fields].ignore`.** Does the global list stay and compose + with per-scope `ignore`, or does it become sugar for a `path = "**"` scope? + Prefer not having two mechanisms for the same thing. +- **Default for unmatched paths.** A file matching no scope should presumably + stay open-world, preserving today's behavior for every existing vault. Confirm + this is the migration story. +- **Which violation kind.** Reuse `Disallowed`, or add a distinct kind + (`Undeclared`?) so the two cases stay legible in the output? `Disallowed` + currently carries the rule string `allowed in [...]`, which would read oddly + for this case. +- **Interaction with auto-update.** Inside a frozen scope, should `check`'s + auto-update refuse to absorb a new field, rather than adding it and then + flagging it? Probably yes, or the two features fight. +- **Does `frozen` belong per-scope or is it the only policy?** If `[[scope]]` is + going to exist, consider what else is directory-scoped (see TODO-0195 for + cross-field rules, which want a similar home). ### Interaction with TODO-0195 -Both this and [TODO-0195](TODO-0195.md) need the same missing thing: **a place in `mdvs.toml` for rules that are not field-local**. Today every rule hangs off exactly one `[[fields.field]]`. Directory policy and cross-field conditions are both cross-cutting. Design the container once, here, so 0195's rules can live in the same section rather than inventing a second syntax. +Both this and [TODO-0195](TODO-0195.md) need the same missing thing: **a place +in `mdvs.toml` for rules that are not field-local**. Today every rule hangs off +exactly one `[[fields.field]]`. Directory policy and cross-field conditions are +both cross-cutting. Design the container once, here, so 0195's rules can live in +the same section rather than inventing a second syntax. ### Documentation debt this closes -`book/src/recipes/ci.md` currently documents the gap explicitly and offers a workaround: +`book/src/recipes/ci.md` currently documents the gap explicitly and offers a +workaround: ```bash mdvs check --no-update --output json | jq -e '.new_fields | length == 0' ``` -That section, and the sentence "A schema-level 'freeze this directory' option does not exist yet", are what this TODO replaces. +That section, and the sentence "A schema-level 'freeze this directory' option +does not exist yet", are what this TODO replaces. ## Files -- `crates/mdvs/src/schema/config.rs` — the `[[scope]]` section, plus a `MdvsToml::validate()` invariant -- `crates/mdvs/src/cmd/check/validate.rs` — the undeclared-key pass (needs to iterate file keys, not just declared fields) +- `crates/mdvs/src/schema/config.rs` — the `[[scope]]` section, plus a + `MdvsToml::validate()` invariant +- `crates/mdvs/src/cmd/check/validate.rs` — the undeclared-key pass (needs to + iterate file keys, not just declared fields) - `crates/mdvs/src/output.rs` — violation kind, if a new one is added -- `book/src/recipes/ci.md`, `book/src/configuration.md`, `book/src/concepts/validation.md` — docs +- `book/src/recipes/ci.md`, `book/src/configuration.md`, + `book/src/concepts/validation.md` — docs diff --git a/docs/spec/todos/TODO-0195.md b/docs/spec/todos/TODO-0195.md index d7a176b..4a766d2 100644 --- a/docs/spec/todos/TODO-0195.md +++ b/docs/spec/todos/TODO-0195.md @@ -13,7 +13,11 @@ related: [194, 156] ## Summary -Every validation rule in mdvs hangs off exactly one `[[fields.field]]`. There is no way to say "`closed_date` is required when `status` is `closed`", nor to model a field whose shape depends on a discriminant (a Rust-style tagged enum). This TODO covers conditional requiredness as the near-term feature, and records the variant-type idea as its expensive endpoint. +Every validation rule in mdvs hangs off exactly one `[[fields.field]]`. There is +no way to say "`closed_date` is required when `status` is `closed`", nor to +model a field whose shape depends on a discriminant (a Rust-style tagged enum). +This TODO covers conditional requiredness as the near-term feature, and records +the variant-type idea as its expensive endpoint. ## Details @@ -21,17 +25,31 @@ Every validation rule in mdvs hangs off exactly one `[[fields.field]]`. There is Two forms, in increasing cost: -**Presence-based.** "If `a` is present, `b` and `c` are required." Maps directly onto JSON Schema's `dependentRequired`, which the `jsonschema` crate already supports. The violation maps onto the existing `ViolationKind::MissingRequired`. No new evaluation machinery. +**Presence-based.** "If `a` is present, `b` and `c` are required." Maps directly +onto JSON Schema's `dependentRequired`, which the `jsonschema` crate already +supports. The violation maps onto the existing `ViolationKind::MissingRequired`. +No new evaluation machinery. -**Value-based.** "When `status == closed`, require `closure_reason`." Needs `if` / `then` / `else`. More expressive, and the one that pairs with Part B — but worse error messages, since a failing `if/then` reports at the subschema level rather than naming the triggering field. +**Value-based.** "When `status == closed`, require `closure_reason`." Needs `if` +/ `then` / `else`. More expressive, and the one that pairs with Part B — but +worse error messages, since a failing `if/then` reports at the subschema level +rather than naming the triggering field. -**The complement is as important as the rule.** Conditional *requiredness* alone is half the constraint. The other half is conditional *disallowal* — `closure_reason` should be meaningless, and rejected, when `status` is `open`. Required-when plus disallowed-when together are what make the constraint actually exhaustive. Ship both directions or the feature is a footgun. +**The complement is as important as the rule.** Conditional _requiredness_ alone +is half the constraint. The other half is conditional _disallowal_ — +`closure_reason` should be meaningless, and rejected, when `status` is `open`. +Required-when plus disallowed-when together are what make the constraint +actually exhaustive. Ship both directions or the feature is a footgun. -`schema/json_schema/validate.rs` currently hard-rejects `oneOf`, `$ref`, `if`/`then` and friends via the curated reject list. That gate needs to open selectively for whichever keywords this lands on — the point of the gate is to reject *unsupported* JSON Schema, not to freeze the supported set. +`schema/json_schema/validate.rs` currently hard-rejects `oneOf`, `$ref`, +`if`/`then` and friends via the curated reject list. That gate needs to open +selectively for whichever keywords this lands on — the point of the gate is to +reject _unsupported_ JSON Schema, not to freeze the supported set. ### Part B — Rust-style variant types -The idea: instead of a flat `status` with three categories, model substates with payloads — +The idea: instead of a flat `status` with three categories, model substates with +payloads — ```yaml status: @@ -40,19 +58,46 @@ status: date: 2027-01-01 ``` -**The file format is not the obstacle.** YAML and TOML both represent tagged unions fine; the shape above is externally-tagged, and `{kind: closed, outcome: ..., date: ...}` is internally-tagged. These are exactly serde's enum representations. The obstacles are three deliberate mdvs decisions: - -1. **The schema gate rejects `oneOf`**, which a tagged union needs (`schema/json_schema/validate.rs`). -2. **Storage assumes a fixed shape.** `index/storage.rs::transpose_to_storage_type` builds one nested Arrow Struct for the whole corpus; a variant means the shape varies per row. A sparse-struct encoding works around this — `status.kind`, `status.closed.date`, `status.rejected.reason`, all nullable, only the active variant populated — and stays SQL-queryable via `--where "status.kind = 'closed'"`. Arrow's native `Union` is the wrong tool: Lance and DataFusion support for it is thin. -3. **Inference.** mdvs infers schemas from the corpus. Deriving "this key always has exactly one of N shapes" is far harder than deriving a scalar type, and `init --force` would likely destroy a hand-written variant declaration. Wave C already rejected `Array(Object{...})` and top-level `Object` on disk for related reasons (see [TODO-0156](TODO-0156.md)). - -**Part A is ~90% of Part B at ~10% of the cost.** The flat encoding of the example above is: `status` with `categories = ["open", "closed"]`, plus "when `status == closed`, require `closure_reason` (itself categorical) and `closed_date`", plus "when `status == open`, disallow both". Same guarantees, no storage change, queryable with plain SQL today. What is genuinely lost is the *type-level* guarantee — nothing structurally prevents someone declaring a fourth field that only makes sense for a variant nobody wrote a rule for. - -Recommendation: build Part A in both directions, live with it, and only revisit Part B if the flat encoding proves insufficient in practice rather than merely inelegant. +**The file format is not the obstacle.** YAML and TOML both represent tagged +unions fine; the shape above is externally-tagged, and +`{kind: closed, outcome: ..., date: ...}` is internally-tagged. These are +exactly serde's enum representations. The obstacles are three deliberate mdvs +decisions: + +1. **The schema gate rejects `oneOf`**, which a tagged union needs + (`schema/json_schema/validate.rs`). +2. **Storage assumes a fixed shape.** + `index/storage.rs::transpose_to_storage_type` builds one nested Arrow Struct + for the whole corpus; a variant means the shape varies per row. A + sparse-struct encoding works around this — `status.kind`, + `status.closed.date`, `status.rejected.reason`, all nullable, only the active + variant populated — and stays SQL-queryable via + `--where "status.kind = 'closed'"`. Arrow's native `Union` is the wrong tool: + Lance and DataFusion support for it is thin. +3. **Inference.** mdvs infers schemas from the corpus. Deriving "this key always + has exactly one of N shapes" is far harder than deriving a scalar type, and + `init --force` would likely destroy a hand-written variant declaration. Wave + C already rejected `Array(Object{...})` and top-level `Object` on disk for + related reasons (see [TODO-0156](TODO-0156.md)). + +**Part A is ~90% of Part B at ~10% of the cost.** The flat encoding of the +example above is: `status` with `categories = ["open", "closed"]`, plus "when +`status == closed`, require `closure_reason` (itself categorical) and +`closed_date`", plus "when `status == open`, disallow both". Same guarantees, no +storage change, queryable with plain SQL today. What is genuinely lost is the +_type-level_ guarantee — nothing structurally prevents someone declaring a +fourth field that only makes sense for a variant nobody wrote a rule for. + +Recommendation: build Part A in both directions, live with it, and only revisit +Part B if the flat encoding proves insufficient in practice rather than merely +inelegant. ### Where these rules live -Cross-field rules do not fit `[[fields.field]]` — by construction they are about a *relationship between* fields. They need their own section, which is the same conclusion [TODO-0194](TODO-0194.md) reaches for directory policy. **Design the container once, in 0194, and put these rules in it.** Hence `depends_on: [194]`. +Cross-field rules do not fit `[[fields.field]]` — by construction they are about +a _relationship between_ fields. They need their own section, which is the same +conclusion [TODO-0194](TODO-0194.md) reaches for directory policy. **Design the +container once, in 0194, and put these rules in it.** Hence `depends_on: [194]`. Sketch, deliberately not settled: @@ -65,15 +110,28 @@ disallow = [] ### Design questions -- Presence-based only for v0, or value-based from the start? Value-based is what the motivating example needs, so presence-only may not be worth shipping alone. -- How does a conditional rule interact with a field's own path-scoped `required` globs? Two mechanisms can now make the same field required; the violation output must stay legible about which one fired. -- Error message quality. A raw `if/then` failure from the `jsonschema` crate will not name the triggering field. Mapping it back to something like "`closure_reason` is required when `status` is `closed`" likely means evaluating the condition ourselves rather than delegating wholesale. -- Does inference ever propose these? Almost certainly not for v0 — the correlation heuristics are a research project. Hand-written only, and `init --force` must not silently drop them (today it clobbers hand-tuned constraints, which is already a known sharp edge). +- Presence-based only for v0, or value-based from the start? Value-based is what + the motivating example needs, so presence-only may not be worth shipping + alone. +- How does a conditional rule interact with a field's own path-scoped `required` + globs? Two mechanisms can now make the same field required; the violation + output must stay legible about which one fired. +- Error message quality. A raw `if/then` failure from the `jsonschema` crate + will not name the triggering field. Mapping it back to something like + "`closure_reason` is required when `status` is `closed`" likely means + evaluating the condition ourselves rather than delegating wholesale. +- Does inference ever propose these? Almost certainly not for v0 — the + correlation heuristics are a research project. Hand-written only, and + `init --force` must not silently drop them (today it clobbers hand-tuned + constraints, which is already a known sharp edge). ## Files -- `crates/mdvs/src/schema/config.rs` — the rule section + `MdvsToml::validate()` invariants -- `crates/mdvs/src/schema/json_schema/validate.rs` — open the gate for the chosen keywords -- `crates/mdvs/src/schema/json_schema/to_canonical.rs` / `from_canonical.rs` — emit and reverse the rules +- `crates/mdvs/src/schema/config.rs` — the rule section + `MdvsToml::validate()` + invariants +- `crates/mdvs/src/schema/json_schema/validate.rs` — open the gate for the + chosen keywords +- `crates/mdvs/src/schema/json_schema/to_canonical.rs` / `from_canonical.rs` — + emit and reverse the rules - `crates/mdvs/src/cmd/check/` — evaluation + violation mapping - `book/src/concepts/validation.md`, `book/src/configuration.md` — docs diff --git a/docs/spec/todos/TODO-0196.md b/docs/spec/todos/TODO-0196.md index ed408e8..a467925 100644 --- a/docs/spec/todos/TODO-0196.md +++ b/docs/spec/todos/TODO-0196.md @@ -13,13 +13,17 @@ related: [194, 195] ## Summary -Two `[[fields.field]]` entries with the same `name` are accepted by config load without warning. Downstream lookup is keyed by the bare name, so the later entry silently overwrites the earlier one and the first declaration is discarded. Reject duplicates at config load. +Two `[[fields.field]]` entries with the same `name` are accepted by config load +without warning. Downstream lookup is keyed by the bare name, so the later entry +silently overwrites the earlier one and the first declaration is discarded. +Reject duplicates at config load. ## Details ### Reproduction -Verified against v0.8.4. Config declaring `status` twice, scoped to disjoint directories: +Verified against v0.8.4. Config declaring `status` twice, scoped to disjoint +directories: ```toml [[fields.field]] @@ -37,7 +41,8 @@ required = ["projects/**"] nullable = false ``` -With `blog/post.md` containing `status: draft` and `projects/p.md` containing `status: 3`: +With `blog/post.md` containing `status: draft` and `projects/p.md` containing +`status: 3`: ``` Checked 2 files — 2 violation(s) @@ -46,7 +51,10 @@ status │ Wrong type │ type Integer │ blog/post.md (got Stri status │ Not allowed │ allowed in ["projects/**"] │ blog/post.md ``` -The `blog/**` declaration is gone. `blog/post.md` is judged against the `projects/**` entry — wrong type *and* wrong path — while `projects/p.md`, the file the surviving rule was never meant to cover alone, passes. Both violations are artifacts of the collapse, not of the vault. +The `blog/**` declaration is gone. `blog/post.md` is judged against the +`projects/**` entry — wrong type _and_ wrong path — while `projects/p.md`, the +file the surviving rule was never meant to cover alone, passes. Both violations +are artifacts of the collapse, not of the vault. ### Cause @@ -61,36 +69,66 @@ let field_map: HashMap<&str, _> = config .collect(); ``` -`collect()` into a `HashMap` keeps the last value for a repeated key. `FieldValidators::build` (`cmd/check/field_meta.rs`) builds a `HashMap` the same way, so the compiled validator collapses identically. +`collect()` into a `HashMap` keeps the last value for a repeated key. +`FieldValidators::build` (`cmd/check/field_meta.rs`) builds a +`HashMap` the same way, so the compiled validator collapses +identically. -`MdvsToml::validate()` has nine invariants (`schema/config.rs`); none concerns duplicate names. Invariant 8 covers *shape* conflicts (a name declared both as a leaf and as a parent of a dotted name) but not two declarations of the same leaf. +`MdvsToml::validate()` has nine invariants (`schema/config.rs`); none concerns +duplicate names. Invariant 8 covers _shape_ conflicts (a name declared both as a +leaf and as a parent of a dotted name) but not two declarations of the same +leaf. ### Fix -Add **invariant 10**: no two `[[fields.field]]` entries may share a `name`. Error at config load, naming the field and pointing at the duplicate, in the style of the existing invariant messages. +Add **invariant 10**: no two `[[fields.field]]` entries may share a `name`. +Error at config load, naming the field and pointing at the duplicate, in the +style of the existing invariant messages. -This is a strict improvement regardless of what happens with directory-scoped policy — it converts a silent wrong answer into a clear error, and it is a small, self-contained change. +This is a strict improvement regardless of what happens with directory-scoped +policy — it converts a silent wrong answer into a clear error, and it is a +small, self-contained change. ### Why the duplicate looks reasonable to write -Someone reaching for this config is trying to express "the same field name means different things in different directories" — `status` as a workflow state under `projects/**` and as a publication state under `blog/**`. The config reads as though scoping should disambiguate, because `allowed` already scopes everything else about a field. +Someone reaching for this config is trying to express "the same field name means +different things in different directories" — `status` as a workflow state under +`projects/**` and as a publication state under `blog/**`. The config reads as +though scoping should disambiguate, because `allowed` already scopes everything +else about a field. -It cannot work as written, for a reason that sits below validation: **field identity is the bare name, and that reaches into storage.** `index/storage.rs` transposes frontmatter into a nested Arrow Struct where `data.status` is one column with one type. Two declarations with two types cannot share it. +It cannot work as written, for a reason that sits below validation: **field +identity is the bare name, and that reaches into storage.** `index/storage.rs` +transposes frontmatter into a nested Arrow Struct where `data.status` is one +column with one type. Two declarations with two types cannot share it. -So rejecting duplicates is correct, but it leaves the underlying need unmet. The proposed resolution, to be designed in [TODO-0194](TODO-0194.md): +So rejecting duplicates is correct, but it leaves the underlying need unmet. The +proposed resolution, to be designed in [TODO-0194](TODO-0194.md): - **One name → one type**, enforced. Type is what storage constrains. -- **Constraints may vary by scope.** `categories`, ranges, lengths, patterns and requiredness are validation-only, so they can differ per directory without touching the column. +- **Constraints may vary by scope.** `categories`, ranges, lengths, patterns and + requiredness are validation-only, so they can differ per directory without + touching the column. -That covers the common real case, where the *values* diverge and the type does not. When the type genuinely differs, the fields are different things and should be named differently. +That covers the common real case, where the _values_ diverge and the type does +not. When the type genuinely differs, the fields are different things and should +be named differently. -Implementing invariant 10 now does not foreclose that: a future scoped-constraint syntax would attach constraint blocks to a single `[[fields.field]]` entry rather than repeating the entry, so "one entry per name" remains true either way. +Implementing invariant 10 now does not foreclose that: a future +scoped-constraint syntax would attach constraint blocks to a single +`[[fields.field]]` entry rather than repeating the entry, so "one entry per +name" remains true either way. ### Open question -Should invariant 10 fire for names differing only by case, or by dotted-path normalisation? Field names are matched exactly elsewhere, so exact-match rejection is the consistent choice, but worth confirming no vault relies on near-duplicate names. +Should invariant 10 fire for names differing only by case, or by dotted-path +normalisation? Field names are matched exactly elsewhere, so exact-match +rejection is the consistent choice, but worth confirming no vault relies on +near-duplicate names. ## Files -- `crates/mdvs/src/schema/config.rs` — invariant 10 in `MdvsToml::validate()`, plus a rejection test alongside the existing invariant tests -- `crates/mdvs/src/cmd/check/validate.rs` — no change required once the invariant lands, but worth a comment noting the map assumes unique names +- `crates/mdvs/src/schema/config.rs` — invariant 10 in `MdvsToml::validate()`, + plus a rejection test alongside the existing invariant tests +- `crates/mdvs/src/cmd/check/validate.rs` — no change required once the + invariant lands, but worth a comment noting the map assumes unique names diff --git a/docs/spec/todos/index.md b/docs/spec/todos/index.md index db310c1..635d905 100644 --- a/docs/spec/todos/index.md +++ b/docs/spec/todos/index.md @@ -1,200 +1,200 @@ # TODOs -| ID | Title | Status | Priority | Created | -|----|-------|--------|----------|---------| -| [0001](TODO-0001.md) | Null-transparent type widening | done | high | 2026-03-02 | -| [0002](TODO-0002.md) | Add .mdvsignore and .gitignore support | done | high | 2026-03-02 | -| [0003](TODO-0003.md) | Fix --auto-build flag on init | done | medium | 2026-03-02 | -| [0004](TODO-0004.md) | Rename --where-clause to --where | done | low | 2026-03-02 | -| [0005](TODO-0005.md) | Differentiate null from absent in check | done | high | 2026-03-02 | -| [0006](TODO-0006.md) | Support categorical constraints on fields | done | medium | 2026-03-02 | -| [0007](TODO-0007.md) | Support Date and DateTime field types | done | high | 2026-03-02 | -| [0008](TODO-0008.md) | Support value boundary constraints on numeric fields | done | high | 2026-03-02 | -| [0009](TODO-0009.md) | Custom field and text processors | todo | low | 2026-03-02 | -| [0010](TODO-0010.md) | Support length constraints on strings and arrays | done | high | 2026-03-02 | -| [0011](TODO-0011.md) | Incremental build | done | medium | 2026-03-02 | -| [0012](TODO-0012.md) | Store build metadata in parquet | done | high | 2026-03-02 | -| [0013](TODO-0013.md) | Search verifies model against parquet metadata | done | high | 2026-03-02 | -| [0014](TODO-0014.md) | Build detects manual config changes via parquet metadata | done | high | 2026-03-02 | -| [0015](TODO-0015.md) | Implement info and clean commands | done | high | 2026-03-02 | -| [0016](TODO-0016.md) | Replace Parquet + DataFusion with Lance + LanceDB (full swap, hybrid search) | done | high | 2026-03-02 | -| [0017](TODO-0017.md) | Ollama embedding provider | todo | medium | 2026-03-02 | -| [0018](TODO-0018.md) | Cloud embedding providers (Azure, AWS Bedrock) | todo | low | 2026-03-02 | -| [0019](TODO-0019.md) | Global --verbose flag | done | medium | 2026-03-02 | -| [0020](TODO-0020.md) | Add required Cargo.toml metadata for crates.io | done | high | 2026-03-03 | -| [0021](TODO-0021.md) | Downgrade edition from 2024 to 2021 | done | high | 2026-03-03 | -| [0022](TODO-0022.md) | Update README for release | done | high | 2026-03-03 | -| [0023](TODO-0023.md) | Clean up stale .gitignore entries | done | low | 2026-03-03 | -| [0024](TODO-0024.md) | Add CHANGELOG.md | done | medium | 2026-03-03 | -| [0025](TODO-0025.md) | Trim published package size | done | low | 2026-03-03 | -| [0026](TODO-0026.md) | Fix clippy collapsible_if warnings | done | low | 2026-03-03 | -| [0027](TODO-0027.md) | Prefix internal parquet columns to avoid frontmatter collisions | done | medium | 2026-03-03 | -| [0028](TODO-0028.md) | Bare field names in --where clauses | done | medium | 2026-03-03 | -| [0029](TODO-0029.md) | User documentation site with mdBook | done | high | 2026-03-03 | -| [0030](TODO-0030.md) | Homebrew tap and prebuilt binaries | superseded | medium | 2026-03-03 | -| [0031](TODO-0031.md) | Example vault repository | done | medium | 2026-03-04 | -| [0032](TODO-0032.md) | Fix verbose tracing output — show timing and useful info | done | high | 2026-03-04 | -| [0033](TODO-0033.md) | Unified output format (umbrella) | done | high | 2026-03-04 | -| [0034](TODO-0034.md) | Flag rework — rename human→text, add --logs, repurpose -v | done | high | 2026-03-05 | -| [0035](TODO-0035.md) | Add tabled + terminal_size and create table style helpers | done | high | 2026-03-05 | -| [0036](TODO-0036.md) | Rewrite clean command output | done | high | 2026-03-05 | -| [0037](TODO-0037.md) | Rewrite search command output | done | high | 2026-03-05 | -| [0038](TODO-0038.md) | Rewrite build command output | done | high | 2026-03-05 | -| [0039](TODO-0039.md) | Rewrite check command output | done | high | 2026-03-05 | -| [0040](TODO-0040.md) | Rewrite init command output | done | high | 2026-03-05 | -| [0041](TODO-0041.md) | Rewrite update command output | done | high | 2026-03-05 | -| [0042](TODO-0042.md) | Rewrite info command output | done | high | 2026-03-05 | -| [0043](TODO-0043.md) | Fix tracing levels — distinct debug/trace events with elapsed times | todo | medium | 2026-03-05 | -| [0044](TODO-0044.md) | Cargo.toml metadata and crate optimization | done | high | 2026-03-06 | -| [0045](TODO-0045.md) | cargo-dist initialization and release workflow | done | high | 2026-03-06 | -| [0046](TODO-0046.md) | Homebrew tap via cargo-dist | todo | medium | 2026-03-06 | -| [0047](TODO-0047.md) | npm binary wrapper via cargo-dist | todo | medium | 2026-03-06 | -| [0048](TODO-0048.md) | README install section update | todo | medium | 2026-03-06 | -| [0049](TODO-0049.md) | GitHub Actions CI workflow | done | high | 2026-03-06 | -| [0050](TODO-0050.md) | Fix String null serialization to Arrow NULL | done | high | 2026-03-07 | -| [0051](TODO-0051.md) | Replace panic! in model loading with error propagation | done | high | 2026-03-07 | -| [0052](TODO-0052.md) | Handle unreadable files gracefully in scan | done | high | 2026-03-07 | -| [0053](TODO-0053.md) | Handle symlink escape in scan strip_prefix | done | high | 2026-03-07 | -| [0054](TODO-0054.md) | Handle invalid glob pattern without panicking | done | high | 2026-03-07 | -| [0055](TODO-0055.md) | Add file size limit to scan | done | medium | 2026-03-07 | -| [0056](TODO-0056.md) | Verify .mdvs/ is not a symlink before clean | done | medium | 2026-03-07 | -| [0057](TODO-0057.md) | Refactor build::run() — extract model loading helper | done | medium | 2026-03-07 | -| [0058](TODO-0058.md) | Replace unwrap on path to_str in search | done | medium | 2026-03-07 | -| [0059](TODO-0059.md) | Replace unwrap on JSON serialization in output | done | medium | 2026-03-07 | -| [0060](TODO-0060.md) | Add tests for update command | done | high | 2026-03-07 | -| [0061](TODO-0061.md) | Add test for build validation abort | done | high | 2026-03-07 | -| [0062](TODO-0062.md) | Add test for null value parquet roundtrip | done | high | 2026-03-07 | -| [0063](TODO-0063.md) | Add search edge case tests | done | high | 2026-03-07 | -| [0064](TODO-0064.md) | Add parquet roundtrip tests for complex types | done | medium | 2026-03-07 | -| [0065](TODO-0065.md) | Add tests for table.rs and output.rs | done | low | 2026-03-07 | -| [0066](TODO-0066.md) | Add YAML nesting depth limit | done | medium | 2026-03-07 | -| [0067](TODO-0067.md) | Add frontmatter field count limit | done | medium | 2026-03-07 | -| [0068](TODO-0068.md) | Add context to parquet read error messages | done | low | 2026-03-07 | -| [0069](TODO-0069.md) | Warn when search verbose chunk text is unavailable | done | low | 2026-03-07 | -| [0070](TODO-0070.md) | Extract shared inference logic from init and update | done | low | 2026-03-07 | -| [0071](TODO-0071.md) | Break up monolithic validate() function | done | low | 2026-03-07 | -| [0072](TODO-0072.md) | Escape special characters in search SQL construction | done | medium | 2026-03-07 | -| [0073](TODO-0073.md) | Build violation output goes to stderr instead of stdout | done | high | 2026-03-07 | -| [0074](TODO-0074.md) | Replace DefaultHasher with stable hash for content_hash | done | low | 2026-03-07 | -| [0075](TODO-0075.md) | Support array containment queries in --where | done | medium | 2026-03-07 | -| [0076](TODO-0076.md) | Ergonomic --where queries for field names with spaces | done | medium | 2026-03-07 | -| [0077](TODO-0077.md) | Filter info command output by field name | todo | medium | 2026-03-08 | -| [0078](TODO-0078.md) | Structured error output for all commands | done | medium | 2026-03-08 | -| [0079](TODO-0079.md) | Core pipeline abstractions | done | medium | 2026-03-08 | -| [0080](TODO-0080.md) | Shared step output structs | done | medium | 2026-03-08 | -| [0081](TODO-0081.md) | Rework check command pipeline | done | medium | 2026-03-08 | -| [0082](TODO-0082.md) | Rework build command pipeline | done | medium | 2026-03-08 | -| [0083](TODO-0083.md) | Rework init command pipeline | done | medium | 2026-03-08 | -| [0084](TODO-0084.md) | Rework update command pipeline | done | medium | 2026-03-08 | -| [0085](TODO-0085.md) | Rework search command pipeline | done | medium | 2026-03-08 | -| [0086](TODO-0086.md) | Rework info command pipeline | done | medium | 2026-03-08 | -| [0087](TODO-0087.md) | Rework clean command pipeline | done | medium | 2026-03-08 | -| [0088](TODO-0088.md) | main.rs error handling and exit codes | paused | medium | 2026-03-08 | -| [0089](TODO-0089.md) | Warn on stale index during search | done (subsumed by 0099) | medium | 2026-03-08 | -| [0090](TODO-0090.md) | Remove check_result from BuildCommandOutput | done | medium | 2026-03-08 | -| [0091](TODO-0091.md) | Consistent output rules for text and JSON formats | done | medium | 2026-03-08 | -| [0092](TODO-0092.md) | Compact JSON output — result-only when no errors | done | medium | 2026-03-08 | -| [0093](TODO-0093.md) | Verbose text output — process step lines on success | done | medium | 2026-03-08 | -| [0094](TODO-0094.md) | Hard error on scan safety limits instead of silent skip | todo | medium | 2026-03-09 | -| [0095](TODO-0095.md) | GitHub Actions workflow for mdBook deployment to GitHub Pages | done | medium | 2026-03-12 | -| [0096](TODO-0096.md) | Change array type display from String[] to Array(String) | done | medium | 2026-03-12 | -| [0097](TODO-0097.md) | Explode nested Object fields into dot-separated leaf keys | done | medium | 2026-03-12 | -| [0098](TODO-0098.md) | Build --force should handle dimension mismatch without requiring clean | done | medium | 2026-03-13 | -| [0099](TODO-0099.md) | Redesign auto-update/auto-build pipeline across commands | done | high | 2026-03-13 | -| [0100](TODO-0100.md) | Redesign text output format for all commands | done | high | 2026-03-13 | -| [0101](TODO-0101.md) | Add markdown output format, rename text to pretty | done | medium | 2026-03-13 | -| [0102](TODO-0102.md) | Write and distribute SKILL.md for end-user projects | todo | medium | 2026-03-14 | -| [0103](TODO-0103.md) | Validate config invariants on mdvs.toml load | done | high | 2026-03-14 | -| [0104](TODO-0104.md) | Redesign internal column naming — move prefix from storage to search view | done | medium | 2026-03-14 | -| [0105](TODO-0105.md) | Test and write CI recipe for mdvs check | todo | medium | 2026-03-14 | -| [0106](TODO-0106.md) | Link graph from internal links and wikilinks | todo | medium | 2026-03-14 | -| [0107](TODO-0107.md) | Pre-commit hook for mdvs check | done | medium | 2026-03-14 | -| [0108](TODO-0108.md) | --set-revision with empty string or "None" should clear the revision | done | low | 2026-03-14 | -| [0109](TODO-0109.md) | Clean up DataFusion error messages in --where | todo | low | 2026-03-14 | -| [0110](TODO-0110.md) | Recursive output architecture — nested process steps | done (superseded by 0119) | high | 2026-03-14 | -| [0111](TODO-0111.md) | Reject unknown fields in mdvs.toml with deny_unknown_fields | done | high | 2026-03-14 | -| [0112](TODO-0112.md) | Document JSON output format in the mdBook | todo | medium | 2026-03-15 | -| [0113](TODO-0113.md) | Progress bar for model download and embedding | todo | medium | 2026-03-16 | -| [0114](TODO-0114.md) | Auto-generate CLI output examples in mdBook with mdbook-cmdrun | todo | medium | 2026-03-17 | -| [0115](TODO-0115.md) | Embed asciinema recordings in mdBook for interactive demos | todo | medium | 2026-03-17 | -| [0116](TODO-0116.md) | Trim DataFusion default features to reduce binary size | todo | medium | 2026-03-17 | -| [0117](TODO-0117.md) | Fix null values skipping Disallowed and NullNotAllowed checks | done | high | 2026-03-17 | -| [0118](TODO-0118.md) | Rework README and book intro to show directory-aware schema | done | high | 2026-03-17 | -| [0119](TODO-0119.md) | Unified Step tree architecture — replace pipeline/command output split | done | high | 2026-03-18 | -| [0120](TODO-0120.md) | Step tree: core types — Step, StepOutcome, StepError | done | high | 2026-03-19 | -| [0121](TODO-0121.md) | Step tree: Block enum and Render trait | done | high | 2026-03-19 | -| [0122](TODO-0122.md) | Step tree: Outcome enums and all outcome structs | done | high | 2026-03-19 | -| [0123](TODO-0123.md) | Step tree: shared formatters (format_text, format_markdown) | done | high | 2026-03-19 | -| [0124](TODO-0124.md) | Step tree: custom Serialize on Step and StepOutcome | done | high | 2026-03-19 | -| [0125](TODO-0125.md) | Step tree: convert clean command | done | high | 2026-03-19 | -| [0126](TODO-0126.md) | Step tree: convert info command | done | high | 2026-03-19 | -| [0127](TODO-0127.md) | Step tree: convert check command | done | high | 2026-03-19 | -| [0128](TODO-0128.md) | Step tree: convert init command | done | high | 2026-03-19 | -| [0129](TODO-0129.md) | Step tree: convert update command | done | high | 2026-03-19 | -| [0130](TODO-0130.md) | Step tree: convert build + search commands | done | high | 2026-03-19 | -| [0131](TODO-0131.md) | Step tree: delete old pipeline, update main.rs, simplify output.rs | done | high | 2026-03-19 | -| [0132](TODO-0132.md) | Macro for compact struct generation (crabtime) | done (subsumed by 0137) | low | 2026-03-19 | -| [0133](TODO-0133.md) | Macro for step pipeline boilerplate (early-return pattern) | done (subsumed by 0139) | low | 2026-03-19 | -| [0134](TODO-0134.md) | Step tree post-migration cleanup | done | medium | 2026-03-20 | -| [0135](TODO-0135.md) | Remove Skipped padding from Step tree error paths | done | medium | 2026-03-20 | -| [0136](TODO-0136.md) | Inline auto-update and auto-build logic to eliminate redundant reads | done | high | 2026-03-20 | -| [0137](TODO-0137.md) | Flatten Step tree into steps + result structure | done | high | 2026-03-21 | -| [0138](TODO-0138.md) | Remove enum variant wrapper from JSON output | done | high | 2026-03-23 | -| [0139](TODO-0139.md) | Unify fail helpers across commands | done | low | 2026-03-23 | -| [0140](TODO-0140.md) | Global --dry-run flag | todo | medium | 2026-03-28 | -| [0141](TODO-0141.md) | Global --quiet flag to suppress output on success | todo | medium | 2026-03-29 | -| [0142](TODO-0142.md) | Fix chunk line numbers to exclude frontmatter | done | high | 2026-03-29 | -| [0143](TODO-0143.md) | Additional constraint kinds — design discussion | todo | low | 2026-03-30 | -| [0144](TODO-0144.md) | Embedded Lua scripting for user-side customization | todo | medium | 2026-03-30 | -| [0145](TODO-0145.md) | Support regex pattern constraints on string fields | done | high | 2026-04-02 | -| [0146](TODO-0146.md) | Update mdBook for categorical constraints and reinfer subcommand | done | high | 2026-04-11 | -| [0147](TODO-0147.md) | Restructure specs as developer code map | done | medium | 2026-04-12 | -| [0148](TODO-0148.md) | Generate llms.txt and llms-full.txt from mdBook in CI | todo | medium | 2026-04-16 | -| [0149](TODO-0149.md) | JSON Schema as canonical schema + preprocessor pipeline | done | high | 2026-04-18 | -| [0150](TODO-0150.md) | Strict mode for inference and validation | todo | low | 2026-04-18 | -| [0151](TODO-0151.md) | Category inference and validation broken for widened-to-String fields | done | high | 2026-04-21 | -| [0152](TODO-0152.md) | Layered internal structure with strict downward dependencies | todo | medium | 2026-05-06 | -| [0153](TODO-0153.md) | Small composable extension crates around `jsonschema` (potential extraction) | deferred | low | 2026-05-06 | -| [0154](TODO-0154.md) | Cache overlay validators by signature for path-scoped validation | todo | medium | 2026-05-11 | -| [0155](TODO-0155.md) | Unify mdvs.toml type syntax with CLI Display (function-style) | done | high | 2026-05-11 | -| [0156](TODO-0156.md) | Represent Array of structured items without violating Wave C's flattening | todo | low | 2026-05-13 | -| [0157](TODO-0157.md) | Incremental ANN index optimize instead of full index rebuild | todo | low | 2026-05-22 | -| [0158](TODO-0158.md) | --where translator does not handle quoted/special-character field names | todo | low | 2026-05-22 | -| [0159](TODO-0159.md) | --where on an Array(Float) field panics inside lance-encoding (mitigated) | done | low | 2026-05-22 | -| [0160](TODO-0160.md) | Time-decay weighting for search results | todo | low | 2026-05-25 | -| [0161](TODO-0161.md) | Polish the asciinema demo — readability, highlighting, persistence | todo | medium | 2026-05-25 | -| [0162](TODO-0162.md) | Multi-format frontmatter — support TOML and JSON alongside YAML | done | high | 2026-05-25 | -| [0163](TODO-0163.md) | Expose gray_matter delimiter / engine knobs to let users customize frontmatter parsing | todo | low | 2026-05-26 | -| [0164](TODO-0164.md) | Support array-length constraints (min_items / max_items) | todo | medium | 2026-05-27 | -| [0165](TODO-0165.md) | Survey unused JSON Schema 2020-12 keywords — decide which to adopt | todo | low | 2026-05-27 | -| [0166](TODO-0166.md) | Benchmark mdvs vs QMD — produce launch-credible numbers | todo | high | 2026-05-27 | -| [0167](TODO-0167.md) | Skip embedding-model load and query embed when search mode is fulltext | done | medium | 2026-05-27 | -| [0168](TODO-0168.md) | Compact-snippet output mode for LLM-friendly search results | todo | medium | 2026-05-27 | -| [0169](TODO-0169.md) | Investigate Lance encoding panic on large markdown corpora | done | medium | 2026-05-28 | -| [0170](TODO-0170.md) | Incremental check cache to keep auto-validation cheap at scale | deferred | low | 2026-05-28 | -| [0171](TODO-0171.md) | Semantic-assisted link authoring: turn similarity into explicit links | todo | medium | 2026-05-30 | -| [0172](TODO-0172.md) | Cheap wins in check::validate — precompile globs, hoist conversions, fast-path validators | done | high | 2026-05-29 | -| [0173](TODO-0173.md) | Incremental Lance writes — stop nuking the index on every build | done | high | 2026-05-29 | -| [0174](TODO-0174.md) | content_hash should cover frontmatter, not just the parsed body | todo | medium | 2026-05-29 | -| [0175](TODO-0175.md) | Strip in-code TODO-NNNN references from src/ and examples/ | todo | low | 2026-05-29 | -| [0176](TODO-0176.md) | Refresh docs for v0.7.0 + sweep pre-existing Parquet→Lance drift | todo | medium | 2026-06-04 | -| [0177](TODO-0177.md) | Rewrite README around the LLM-curated knowledge base audience | done | high | 2026-06-04 | -| [0178](TODO-0178.md) | Build an LLM-curated-KB demo (asciinema + script) | todo | high | 2026-06-04 | -| [0179](TODO-0179.md) | Module split — no production file over ~600 lines in crates/mdvs/src | done | high | 2026-06-04 | -| [0180](TODO-0180.md) | Audit non-test code paths for unwrap / panic / expect-without-message | done | high | 2026-06-04 | -| [0181](TODO-0181.md) | Justify or remove unused abstraction headroom (Stage 1/3 enums, x-mdvs.definitions, dead invariants) | todo | medium | 2026-06-04 | -| [0182](TODO-0182.md) | Prune stale and useless pages from docs/spec/ | done | medium | 2026-06-04 | -| [0183](TODO-0183.md) | Trim CLAUDE.md to project-specific rules | done | medium | 2026-06-04 | -| [0184](TODO-0184.md) | Introduce MockEmbedder; keep real-model tests in a separate local-only lane | done | high | 2026-06-05 | -| [0185](TODO-0185.md) | Refresh crates/mdvs/skills/mdvs/SKILL.md for v0.7.0 surface | done | high | 2026-06-05 | -| [0186](TODO-0186.md) | mdvs explain — path-scoped schema query for agent callers | todo | high | 2026-06-06 | -| [0187](TODO-0187.md) | Recipe — agent harness hook for auto-check (and optional auto-explain / pre-validate) | done | medium | 2026-06-06 | -| [0188](TODO-0188.md) | mdvs check --stdin — in-memory frontmatter validation | todo | medium | 2026-06-06 | -| [0189](TODO-0189.md) | Collapse main.rs output dispatch into CommandResult::render() | done | medium | 2026-06-16 | -| [0190](TODO-0190.md) | Design `mdvs scaffold` — unified agent-harness integration command surface | done | high | 2026-06-22 | -| [0191](TODO-0191.md) | Auto-rewrite array-field comparisons in `--where` (parser-based, with translation note) | done | medium | 2026-06-23 | -| [0192](TODO-0192.md) | Don't persist the mock-embedder default to `mdvs.toml` | done | high | 2026-06-23 | -| [0193](TODO-0193.md) | Support .mdx files — free validation, gated search-body stripping | todo | medium | 2026-07-06 | -| [0194](TODO-0194.md) | Directory-scoped schema policy — freeze undeclared fields per path | todo | high | 2026-08-17 | -| [0195](TODO-0195.md) | Cross-field rules — conditional requiredness, and the variant-type question | todo | medium | 2026-08-17 | -| [0196](TODO-0196.md) | Duplicate `[[fields.field]]` names are silently accepted — last one wins | todo | high | 2026-08-20 | +| ID | Title | Status | Priority | Created | +| -------------------- | ---------------------------------------------------------------------------------------------------- | ------------------------- | -------- | ---------- | +| [0001](TODO-0001.md) | Null-transparent type widening | done | high | 2026-03-02 | +| [0002](TODO-0002.md) | Add .mdvsignore and .gitignore support | done | high | 2026-03-02 | +| [0003](TODO-0003.md) | Fix --auto-build flag on init | done | medium | 2026-03-02 | +| [0004](TODO-0004.md) | Rename --where-clause to --where | done | low | 2026-03-02 | +| [0005](TODO-0005.md) | Differentiate null from absent in check | done | high | 2026-03-02 | +| [0006](TODO-0006.md) | Support categorical constraints on fields | done | medium | 2026-03-02 | +| [0007](TODO-0007.md) | Support Date and DateTime field types | done | high | 2026-03-02 | +| [0008](TODO-0008.md) | Support value boundary constraints on numeric fields | done | high | 2026-03-02 | +| [0009](TODO-0009.md) | Custom field and text processors | todo | low | 2026-03-02 | +| [0010](TODO-0010.md) | Support length constraints on strings and arrays | done | high | 2026-03-02 | +| [0011](TODO-0011.md) | Incremental build | done | medium | 2026-03-02 | +| [0012](TODO-0012.md) | Store build metadata in parquet | done | high | 2026-03-02 | +| [0013](TODO-0013.md) | Search verifies model against parquet metadata | done | high | 2026-03-02 | +| [0014](TODO-0014.md) | Build detects manual config changes via parquet metadata | done | high | 2026-03-02 | +| [0015](TODO-0015.md) | Implement info and clean commands | done | high | 2026-03-02 | +| [0016](TODO-0016.md) | Replace Parquet + DataFusion with Lance + LanceDB (full swap, hybrid search) | done | high | 2026-03-02 | +| [0017](TODO-0017.md) | Ollama embedding provider | todo | medium | 2026-03-02 | +| [0018](TODO-0018.md) | Cloud embedding providers (Azure, AWS Bedrock) | todo | low | 2026-03-02 | +| [0019](TODO-0019.md) | Global --verbose flag | done | medium | 2026-03-02 | +| [0020](TODO-0020.md) | Add required Cargo.toml metadata for crates.io | done | high | 2026-03-03 | +| [0021](TODO-0021.md) | Downgrade edition from 2024 to 2021 | done | high | 2026-03-03 | +| [0022](TODO-0022.md) | Update README for release | done | high | 2026-03-03 | +| [0023](TODO-0023.md) | Clean up stale .gitignore entries | done | low | 2026-03-03 | +| [0024](TODO-0024.md) | Add CHANGELOG.md | done | medium | 2026-03-03 | +| [0025](TODO-0025.md) | Trim published package size | done | low | 2026-03-03 | +| [0026](TODO-0026.md) | Fix clippy collapsible_if warnings | done | low | 2026-03-03 | +| [0027](TODO-0027.md) | Prefix internal parquet columns to avoid frontmatter collisions | done | medium | 2026-03-03 | +| [0028](TODO-0028.md) | Bare field names in --where clauses | done | medium | 2026-03-03 | +| [0029](TODO-0029.md) | User documentation site with mdBook | done | high | 2026-03-03 | +| [0030](TODO-0030.md) | Homebrew tap and prebuilt binaries | superseded | medium | 2026-03-03 | +| [0031](TODO-0031.md) | Example vault repository | done | medium | 2026-03-04 | +| [0032](TODO-0032.md) | Fix verbose tracing output — show timing and useful info | done | high | 2026-03-04 | +| [0033](TODO-0033.md) | Unified output format (umbrella) | done | high | 2026-03-04 | +| [0034](TODO-0034.md) | Flag rework — rename human→text, add --logs, repurpose -v | done | high | 2026-03-05 | +| [0035](TODO-0035.md) | Add tabled + terminal_size and create table style helpers | done | high | 2026-03-05 | +| [0036](TODO-0036.md) | Rewrite clean command output | done | high | 2026-03-05 | +| [0037](TODO-0037.md) | Rewrite search command output | done | high | 2026-03-05 | +| [0038](TODO-0038.md) | Rewrite build command output | done | high | 2026-03-05 | +| [0039](TODO-0039.md) | Rewrite check command output | done | high | 2026-03-05 | +| [0040](TODO-0040.md) | Rewrite init command output | done | high | 2026-03-05 | +| [0041](TODO-0041.md) | Rewrite update command output | done | high | 2026-03-05 | +| [0042](TODO-0042.md) | Rewrite info command output | done | high | 2026-03-05 | +| [0043](TODO-0043.md) | Fix tracing levels — distinct debug/trace events with elapsed times | todo | medium | 2026-03-05 | +| [0044](TODO-0044.md) | Cargo.toml metadata and crate optimization | done | high | 2026-03-06 | +| [0045](TODO-0045.md) | cargo-dist initialization and release workflow | done | high | 2026-03-06 | +| [0046](TODO-0046.md) | Homebrew tap via cargo-dist | todo | medium | 2026-03-06 | +| [0047](TODO-0047.md) | npm binary wrapper via cargo-dist | todo | medium | 2026-03-06 | +| [0048](TODO-0048.md) | README install section update | todo | medium | 2026-03-06 | +| [0049](TODO-0049.md) | GitHub Actions CI workflow | done | high | 2026-03-06 | +| [0050](TODO-0050.md) | Fix String null serialization to Arrow NULL | done | high | 2026-03-07 | +| [0051](TODO-0051.md) | Replace panic! in model loading with error propagation | done | high | 2026-03-07 | +| [0052](TODO-0052.md) | Handle unreadable files gracefully in scan | done | high | 2026-03-07 | +| [0053](TODO-0053.md) | Handle symlink escape in scan strip_prefix | done | high | 2026-03-07 | +| [0054](TODO-0054.md) | Handle invalid glob pattern without panicking | done | high | 2026-03-07 | +| [0055](TODO-0055.md) | Add file size limit to scan | done | medium | 2026-03-07 | +| [0056](TODO-0056.md) | Verify .mdvs/ is not a symlink before clean | done | medium | 2026-03-07 | +| [0057](TODO-0057.md) | Refactor build::run() — extract model loading helper | done | medium | 2026-03-07 | +| [0058](TODO-0058.md) | Replace unwrap on path to_str in search | done | medium | 2026-03-07 | +| [0059](TODO-0059.md) | Replace unwrap on JSON serialization in output | done | medium | 2026-03-07 | +| [0060](TODO-0060.md) | Add tests for update command | done | high | 2026-03-07 | +| [0061](TODO-0061.md) | Add test for build validation abort | done | high | 2026-03-07 | +| [0062](TODO-0062.md) | Add test for null value parquet roundtrip | done | high | 2026-03-07 | +| [0063](TODO-0063.md) | Add search edge case tests | done | high | 2026-03-07 | +| [0064](TODO-0064.md) | Add parquet roundtrip tests for complex types | done | medium | 2026-03-07 | +| [0065](TODO-0065.md) | Add tests for table.rs and output.rs | done | low | 2026-03-07 | +| [0066](TODO-0066.md) | Add YAML nesting depth limit | done | medium | 2026-03-07 | +| [0067](TODO-0067.md) | Add frontmatter field count limit | done | medium | 2026-03-07 | +| [0068](TODO-0068.md) | Add context to parquet read error messages | done | low | 2026-03-07 | +| [0069](TODO-0069.md) | Warn when search verbose chunk text is unavailable | done | low | 2026-03-07 | +| [0070](TODO-0070.md) | Extract shared inference logic from init and update | done | low | 2026-03-07 | +| [0071](TODO-0071.md) | Break up monolithic validate() function | done | low | 2026-03-07 | +| [0072](TODO-0072.md) | Escape special characters in search SQL construction | done | medium | 2026-03-07 | +| [0073](TODO-0073.md) | Build violation output goes to stderr instead of stdout | done | high | 2026-03-07 | +| [0074](TODO-0074.md) | Replace DefaultHasher with stable hash for content_hash | done | low | 2026-03-07 | +| [0075](TODO-0075.md) | Support array containment queries in --where | done | medium | 2026-03-07 | +| [0076](TODO-0076.md) | Ergonomic --where queries for field names with spaces | done | medium | 2026-03-07 | +| [0077](TODO-0077.md) | Filter info command output by field name | todo | medium | 2026-03-08 | +| [0078](TODO-0078.md) | Structured error output for all commands | done | medium | 2026-03-08 | +| [0079](TODO-0079.md) | Core pipeline abstractions | done | medium | 2026-03-08 | +| [0080](TODO-0080.md) | Shared step output structs | done | medium | 2026-03-08 | +| [0081](TODO-0081.md) | Rework check command pipeline | done | medium | 2026-03-08 | +| [0082](TODO-0082.md) | Rework build command pipeline | done | medium | 2026-03-08 | +| [0083](TODO-0083.md) | Rework init command pipeline | done | medium | 2026-03-08 | +| [0084](TODO-0084.md) | Rework update command pipeline | done | medium | 2026-03-08 | +| [0085](TODO-0085.md) | Rework search command pipeline | done | medium | 2026-03-08 | +| [0086](TODO-0086.md) | Rework info command pipeline | done | medium | 2026-03-08 | +| [0087](TODO-0087.md) | Rework clean command pipeline | done | medium | 2026-03-08 | +| [0088](TODO-0088.md) | main.rs error handling and exit codes | paused | medium | 2026-03-08 | +| [0089](TODO-0089.md) | Warn on stale index during search | done (subsumed by 0099) | medium | 2026-03-08 | +| [0090](TODO-0090.md) | Remove check_result from BuildCommandOutput | done | medium | 2026-03-08 | +| [0091](TODO-0091.md) | Consistent output rules for text and JSON formats | done | medium | 2026-03-08 | +| [0092](TODO-0092.md) | Compact JSON output — result-only when no errors | done | medium | 2026-03-08 | +| [0093](TODO-0093.md) | Verbose text output — process step lines on success | done | medium | 2026-03-08 | +| [0094](TODO-0094.md) | Hard error on scan safety limits instead of silent skip | todo | medium | 2026-03-09 | +| [0095](TODO-0095.md) | GitHub Actions workflow for mdBook deployment to GitHub Pages | done | medium | 2026-03-12 | +| [0096](TODO-0096.md) | Change array type display from String[] to Array(String) | done | medium | 2026-03-12 | +| [0097](TODO-0097.md) | Explode nested Object fields into dot-separated leaf keys | done | medium | 2026-03-12 | +| [0098](TODO-0098.md) | Build --force should handle dimension mismatch without requiring clean | done | medium | 2026-03-13 | +| [0099](TODO-0099.md) | Redesign auto-update/auto-build pipeline across commands | done | high | 2026-03-13 | +| [0100](TODO-0100.md) | Redesign text output format for all commands | done | high | 2026-03-13 | +| [0101](TODO-0101.md) | Add markdown output format, rename text to pretty | done | medium | 2026-03-13 | +| [0102](TODO-0102.md) | Write and distribute SKILL.md for end-user projects | todo | medium | 2026-03-14 | +| [0103](TODO-0103.md) | Validate config invariants on mdvs.toml load | done | high | 2026-03-14 | +| [0104](TODO-0104.md) | Redesign internal column naming — move prefix from storage to search view | done | medium | 2026-03-14 | +| [0105](TODO-0105.md) | Test and write CI recipe for mdvs check | todo | medium | 2026-03-14 | +| [0106](TODO-0106.md) | Link graph from internal links and wikilinks | todo | medium | 2026-03-14 | +| [0107](TODO-0107.md) | Pre-commit hook for mdvs check | done | medium | 2026-03-14 | +| [0108](TODO-0108.md) | --set-revision with empty string or "None" should clear the revision | done | low | 2026-03-14 | +| [0109](TODO-0109.md) | Clean up DataFusion error messages in --where | todo | low | 2026-03-14 | +| [0110](TODO-0110.md) | Recursive output architecture — nested process steps | done (superseded by 0119) | high | 2026-03-14 | +| [0111](TODO-0111.md) | Reject unknown fields in mdvs.toml with deny_unknown_fields | done | high | 2026-03-14 | +| [0112](TODO-0112.md) | Document JSON output format in the mdBook | todo | medium | 2026-03-15 | +| [0113](TODO-0113.md) | Progress bar for model download and embedding | todo | medium | 2026-03-16 | +| [0114](TODO-0114.md) | Auto-generate CLI output examples in mdBook with mdbook-cmdrun | todo | medium | 2026-03-17 | +| [0115](TODO-0115.md) | Embed asciinema recordings in mdBook for interactive demos | todo | medium | 2026-03-17 | +| [0116](TODO-0116.md) | Trim DataFusion default features to reduce binary size | todo | medium | 2026-03-17 | +| [0117](TODO-0117.md) | Fix null values skipping Disallowed and NullNotAllowed checks | done | high | 2026-03-17 | +| [0118](TODO-0118.md) | Rework README and book intro to show directory-aware schema | done | high | 2026-03-17 | +| [0119](TODO-0119.md) | Unified Step tree architecture — replace pipeline/command output split | done | high | 2026-03-18 | +| [0120](TODO-0120.md) | Step tree: core types — Step, StepOutcome, StepError | done | high | 2026-03-19 | +| [0121](TODO-0121.md) | Step tree: Block enum and Render trait | done | high | 2026-03-19 | +| [0122](TODO-0122.md) | Step tree: Outcome enums and all outcome structs | done | high | 2026-03-19 | +| [0123](TODO-0123.md) | Step tree: shared formatters (format_text, format_markdown) | done | high | 2026-03-19 | +| [0124](TODO-0124.md) | Step tree: custom Serialize on Step and StepOutcome | done | high | 2026-03-19 | +| [0125](TODO-0125.md) | Step tree: convert clean command | done | high | 2026-03-19 | +| [0126](TODO-0126.md) | Step tree: convert info command | done | high | 2026-03-19 | +| [0127](TODO-0127.md) | Step tree: convert check command | done | high | 2026-03-19 | +| [0128](TODO-0128.md) | Step tree: convert init command | done | high | 2026-03-19 | +| [0129](TODO-0129.md) | Step tree: convert update command | done | high | 2026-03-19 | +| [0130](TODO-0130.md) | Step tree: convert build + search commands | done | high | 2026-03-19 | +| [0131](TODO-0131.md) | Step tree: delete old pipeline, update main.rs, simplify output.rs | done | high | 2026-03-19 | +| [0132](TODO-0132.md) | Macro for compact struct generation (crabtime) | done (subsumed by 0137) | low | 2026-03-19 | +| [0133](TODO-0133.md) | Macro for step pipeline boilerplate (early-return pattern) | done (subsumed by 0139) | low | 2026-03-19 | +| [0134](TODO-0134.md) | Step tree post-migration cleanup | done | medium | 2026-03-20 | +| [0135](TODO-0135.md) | Remove Skipped padding from Step tree error paths | done | medium | 2026-03-20 | +| [0136](TODO-0136.md) | Inline auto-update and auto-build logic to eliminate redundant reads | done | high | 2026-03-20 | +| [0137](TODO-0137.md) | Flatten Step tree into steps + result structure | done | high | 2026-03-21 | +| [0138](TODO-0138.md) | Remove enum variant wrapper from JSON output | done | high | 2026-03-23 | +| [0139](TODO-0139.md) | Unify fail helpers across commands | done | low | 2026-03-23 | +| [0140](TODO-0140.md) | Global --dry-run flag | todo | medium | 2026-03-28 | +| [0141](TODO-0141.md) | Global --quiet flag to suppress output on success | todo | medium | 2026-03-29 | +| [0142](TODO-0142.md) | Fix chunk line numbers to exclude frontmatter | done | high | 2026-03-29 | +| [0143](TODO-0143.md) | Additional constraint kinds — design discussion | todo | low | 2026-03-30 | +| [0144](TODO-0144.md) | Embedded Lua scripting for user-side customization | todo | medium | 2026-03-30 | +| [0145](TODO-0145.md) | Support regex pattern constraints on string fields | done | high | 2026-04-02 | +| [0146](TODO-0146.md) | Update mdBook for categorical constraints and reinfer subcommand | done | high | 2026-04-11 | +| [0147](TODO-0147.md) | Restructure specs as developer code map | done | medium | 2026-04-12 | +| [0148](TODO-0148.md) | Generate llms.txt and llms-full.txt from mdBook in CI | todo | medium | 2026-04-16 | +| [0149](TODO-0149.md) | JSON Schema as canonical schema + preprocessor pipeline | done | high | 2026-04-18 | +| [0150](TODO-0150.md) | Strict mode for inference and validation | todo | low | 2026-04-18 | +| [0151](TODO-0151.md) | Category inference and validation broken for widened-to-String fields | done | high | 2026-04-21 | +| [0152](TODO-0152.md) | Layered internal structure with strict downward dependencies | todo | medium | 2026-05-06 | +| [0153](TODO-0153.md) | Small composable extension crates around `jsonschema` (potential extraction) | deferred | low | 2026-05-06 | +| [0154](TODO-0154.md) | Cache overlay validators by signature for path-scoped validation | todo | medium | 2026-05-11 | +| [0155](TODO-0155.md) | Unify mdvs.toml type syntax with CLI Display (function-style) | done | high | 2026-05-11 | +| [0156](TODO-0156.md) | Represent Array of structured items without violating Wave C's flattening | todo | low | 2026-05-13 | +| [0157](TODO-0157.md) | Incremental ANN index optimize instead of full index rebuild | todo | low | 2026-05-22 | +| [0158](TODO-0158.md) | --where translator does not handle quoted/special-character field names | todo | low | 2026-05-22 | +| [0159](TODO-0159.md) | --where on an Array(Float) field panics inside lance-encoding (mitigated) | done | low | 2026-05-22 | +| [0160](TODO-0160.md) | Time-decay weighting for search results | todo | low | 2026-05-25 | +| [0161](TODO-0161.md) | Polish the asciinema demo — readability, highlighting, persistence | todo | medium | 2026-05-25 | +| [0162](TODO-0162.md) | Multi-format frontmatter — support TOML and JSON alongside YAML | done | high | 2026-05-25 | +| [0163](TODO-0163.md) | Expose gray_matter delimiter / engine knobs to let users customize frontmatter parsing | todo | low | 2026-05-26 | +| [0164](TODO-0164.md) | Support array-length constraints (min_items / max_items) | todo | medium | 2026-05-27 | +| [0165](TODO-0165.md) | Survey unused JSON Schema 2020-12 keywords — decide which to adopt | todo | low | 2026-05-27 | +| [0166](TODO-0166.md) | Benchmark mdvs vs QMD — produce launch-credible numbers | todo | high | 2026-05-27 | +| [0167](TODO-0167.md) | Skip embedding-model load and query embed when search mode is fulltext | done | medium | 2026-05-27 | +| [0168](TODO-0168.md) | Compact-snippet output mode for LLM-friendly search results | todo | medium | 2026-05-27 | +| [0169](TODO-0169.md) | Investigate Lance encoding panic on large markdown corpora | done | medium | 2026-05-28 | +| [0170](TODO-0170.md) | Incremental check cache to keep auto-validation cheap at scale | deferred | low | 2026-05-28 | +| [0171](TODO-0171.md) | Semantic-assisted link authoring: turn similarity into explicit links | todo | medium | 2026-05-30 | +| [0172](TODO-0172.md) | Cheap wins in check::validate — precompile globs, hoist conversions, fast-path validators | done | high | 2026-05-29 | +| [0173](TODO-0173.md) | Incremental Lance writes — stop nuking the index on every build | done | high | 2026-05-29 | +| [0174](TODO-0174.md) | content_hash should cover frontmatter, not just the parsed body | todo | medium | 2026-05-29 | +| [0175](TODO-0175.md) | Strip in-code TODO-NNNN references from src/ and examples/ | todo | low | 2026-05-29 | +| [0176](TODO-0176.md) | Refresh docs for v0.7.0 + sweep pre-existing Parquet→Lance drift | todo | medium | 2026-06-04 | +| [0177](TODO-0177.md) | Rewrite README around the LLM-curated knowledge base audience | done | high | 2026-06-04 | +| [0178](TODO-0178.md) | Build an LLM-curated-KB demo (asciinema + script) | todo | high | 2026-06-04 | +| [0179](TODO-0179.md) | Module split — no production file over ~600 lines in crates/mdvs/src | done | high | 2026-06-04 | +| [0180](TODO-0180.md) | Audit non-test code paths for unwrap / panic / expect-without-message | done | high | 2026-06-04 | +| [0181](TODO-0181.md) | Justify or remove unused abstraction headroom (Stage 1/3 enums, x-mdvs.definitions, dead invariants) | todo | medium | 2026-06-04 | +| [0182](TODO-0182.md) | Prune stale and useless pages from docs/spec/ | done | medium | 2026-06-04 | +| [0183](TODO-0183.md) | Trim CLAUDE.md to project-specific rules | done | medium | 2026-06-04 | +| [0184](TODO-0184.md) | Introduce MockEmbedder; keep real-model tests in a separate local-only lane | done | high | 2026-06-05 | +| [0185](TODO-0185.md) | Refresh crates/mdvs/skills/mdvs/SKILL.md for v0.7.0 surface | done | high | 2026-06-05 | +| [0186](TODO-0186.md) | mdvs explain — path-scoped schema query for agent callers | todo | high | 2026-06-06 | +| [0187](TODO-0187.md) | Recipe — agent harness hook for auto-check (and optional auto-explain / pre-validate) | done | medium | 2026-06-06 | +| [0188](TODO-0188.md) | mdvs check --stdin — in-memory frontmatter validation | todo | medium | 2026-06-06 | +| [0189](TODO-0189.md) | Collapse main.rs output dispatch into CommandResult::render() | done | medium | 2026-06-16 | +| [0190](TODO-0190.md) | Design `mdvs scaffold` — unified agent-harness integration command surface | done | high | 2026-06-22 | +| [0191](TODO-0191.md) | Auto-rewrite array-field comparisons in `--where` (parser-based, with translation note) | done | medium | 2026-06-23 | +| [0192](TODO-0192.md) | Don't persist the mock-embedder default to `mdvs.toml` | done | high | 2026-06-23 | +| [0193](TODO-0193.md) | Support .mdx files — free validation, gated search-body stripping | todo | medium | 2026-07-06 | +| [0194](TODO-0194.md) | Directory-scoped schema policy — freeze undeclared fields per path | todo | high | 2026-08-17 | +| [0195](TODO-0195.md) | Cross-field rules — conditional requiredness, and the variant-type question | todo | medium | 2026-08-17 | +| [0196](TODO-0196.md) | Duplicate `[[fields.field]]` names are silently accepted — last one wins | todo | high | 2026-08-20 |