diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef7fea3..176a846 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,6 +112,90 @@ jobs: - name: Asset adapter tests run: python scripts/run_acceptance.py --only verify_asset_adapters + # -- document ingestion (v2 Phase 3) -------------------------------- + # Cheapest first again: the parser SPI and the parsers run in under a + # second, so a broken probe or a regressed parser fails immediately + # instead of after the ingest and CLI suites. + - name: Document parser registry tests + run: python scripts/run_acceptance.py --only verify_document_registry + + - name: Document parser tests + run: python scripts/run_acceptance.py --only verify_document_parsers + + - name: Document security tests + run: python scripts/run_acceptance.py --only verify_document_security + + - name: Document ingestion tests + run: python scripts/run_acceptance.py --only verify_document_ingest + + - name: Document search tests + run: python scripts/run_acceptance.py --only verify_document_search + + - name: Document CLI tests + run: python scripts/run_acceptance.py --only verify_document_cli + + - name: Document adapter tests + run: python scripts/run_acceptance.py --only verify_document_adapters + + # The generated fixtures must regenerate to the same DOCUMENT, so an + # "unchanged" document never mints a new Revision and the incremental + # guarantee stays testable. + # + # The comparison differs by format, because the achievable guarantee does: + # + # PDFs - written byte by byte by the generator, no compressor involved, + # so they are compared as exact bytes. + # OOXML - a DOCX/XLSX is a DEFLATE archive, and the compressed bytes + # depend on the platform's zlib build. Comparing the container + # would fail on Linux for fixtures generated on Windows even + # when every part is identical. So the archive is unpacked and + # its member map is compared, which is the property that + # actually matters and catches real generator or library drift. + - name: Document fixtures regenerate to identical content + run: | + python - <<'PY' + import pathlib, subprocess, sys, zipfile + + FIXTURES = pathlib.Path("fixtures/documents") + OOXML = {".docx", ".xlsx"} + + def content(path): + if path.suffix.lower() in OOXML: + with zipfile.ZipFile(path) as zf: + return {i.filename: zf.read(i.filename) + for i in zf.infolist()} + return path.read_bytes() + + before = {p.name: content(p) for p in sorted(FIXTURES.glob("sample-*"))} + assert before, "no fixtures found" + subprocess.run([sys.executable, "scripts/build_document_fixtures.py"], + check=True, capture_output=True) + after = {p.name: content(p) for p in sorted(FIXTURES.glob("sample-*"))} + + problems = [] + for name, old in before.items(): + new = after.get(name) + if new == old: + continue + if isinstance(old, dict) and isinstance(new, dict): + # Name the exact differing part, so the failure is actionable + # rather than just "the bytes changed". + added = sorted(set(new) - set(old)) + removed = sorted(set(old) - set(new)) + changed = sorted(k for k in set(old) & set(new) + if old[k] != new[k]) + problems.append(f"{name}: added={added} removed={removed} " + f"changed={changed}") + else: + problems.append(f"{name}: bytes differ " + f"({len(old)} -> {len(new or b'')})") + assert not problems, "fixtures are not reproducible:\n " + \ + "\n ".join(problems) + print("document fixtures reproducible:", len(before), "files " + f"({sum(1 for n in before if n.endswith(tuple(OOXML)))} OOXML " + "compared by member content, the rest byte for byte)") + PY + - name: MCP smoke test run: | python - <<'PY' @@ -122,16 +206,28 @@ jobs: from openmind.runtime import get_runtime # The nine core tools are a STABLE external contract; Phase 2 adds - # read-only Asset tools ALONGSIDE them, never in place of one. + # read-only Asset tools and Phase 3 read-only document tools + # ALONGSIDE them, never in place of one. core = {"search", "route", "dispatch", "get_glossary", "find_similar_cases", "save_case", "get_doc", "propose_fix", "apply_fix"} asset = {"list_assets", "get_asset", "get_asset_revisions", "get_evidence"} + document = {"list_documents", "get_document", "get_document_outline", + "search_documents", "search_knowledge", + "find_document_related_candidates"} server = mcp_server.create_mcp_server(get_runtime()) names = {t.name for t in asyncio.run(server.list_tools())} assert core <= names, f"core MCP tool missing: {core - names}" assert asset <= names, f"asset MCP tool missing: {asset - names}" - assert names == core | asset, f"unexpected MCP tools: {names ^ (core | asset)}" + assert document <= names, f"document MCP tool missing: {document - names}" + expected = core | asset | document + assert names == expected, f"unexpected MCP tools: {names ^ expected}" + # Phase 3 adds NO document-write tool: importing reads a local file, + # and exposing that over MCP would let a client make the server read + # a path it chose. + assert not {n for n in names + if n.startswith(("add_", "import_", "create_", + "delete_", "update_"))}, names assert server.name == "open-mind", server.name print("MCP smoke OK:", len(names), "tools") PY @@ -227,7 +323,7 @@ jobs: - name: Artifact export contract run: python tests/verify_artifacts.py - - name: Migration to v0003 + content-store byte round-trip + - name: Migration to v0004 + content-store byte round-trip shell: bash run: | python - <<'PY' @@ -236,7 +332,15 @@ jobs: os.environ["OPENMIND_MACHINE_DIR"] = tempfile.mkdtemp() from openmind import db, content_store as cs db.init_db() - assert db.migration_status()["version"] == 3, db.migration_status() + assert db.migration_status()["version"] == 4, db.migration_status() + conn = db._c() + tables = {r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'")} + assert {"document_parses", "document_index"} <= tables, tables + assert "content_blob_hash" in { + r[1] for r in conn.execute("PRAGMA table_info(segments)")} + assert "payload_json" in { + r[1] for r in conn.execute("PRAGMA table_info(jobs)")} # content-addressed blob byte round-trip + reuse + SHA-256 identity import hashlib data = b"phase2 content \x00\x01\x02 bytes" @@ -266,18 +370,110 @@ jobs: [sys.executable, "-m", "openmind.cli", *args])) data = cli("asset", "list", "--workspace", ws, "--json") assert data["ok"] and data["total"] >= 1, data - aid = data["assets"][0]["id"] + # This step proves the CODE asset chain, so pick a code asset + # explicitly. Since v2 Phase 3 the fixture's README.md and + # docs/GLOSSARY.md are also ingested (as DOCUMENT assets), and they + # sort first — taking assets[0] would silently retarget this check. + code = [a for a in data["assets"] if a["asset_type"] == "source-code"] + assert code, data + aid = code[0]["id"] show = cli("asset", "show", "--workspace", ws, "--asset", aid, "--json") rid = show["asset"]["current_revision"]["id"] segs = cli("asset", "segments", "--workspace", ws, "--revision", rid, "--json") eid = segs["segments"][0]["evidence_id"] ev = cli("asset", "evidence", "--workspace", ws, "--evidence", eid, "--json") assert ev["snapshot"]["status"] == "available", ev + assert ev["locator"]["kind"] == "source-range", ev assert not ev["locator"]["file"].startswith("/"), ev + + # The portable-key rule holds for EVERY asset, whichever locator kind + # it uses: a code locator names its target in `file`, a document + # locator in `document`, and neither may ever be absolute. + for asset in data["assets"]: + rev = cli("asset", "show", "--workspace", ws, + "--asset", asset["id"], "--json")["asset"] + current = rev.get("current_revision") + if not current: + continue + seg = cli("asset", "segments", "--workspace", ws, + "--revision", current["id"], "--limit", "1", "--json") + evidence_id = seg["segments"][0]["evidence_id"] + locator = cli("asset", "evidence", "--workspace", ws, + "--evidence", evidence_id, "--json")["locator"] + key = locator.get("file") or locator.get("document") or "" + assert key, (asset["logical_key"], locator) + assert not key.startswith("/") and ":" not in key[:3], \ + (asset["logical_key"], locator) print("asset CLI smoke OK:", data["total"], "assets, evidence snapshot", ev["snapshot"]["status"]) PY + # -- document ingestion smoke (v2 Phase 3) -------------------------- + # One document of each shape that matters on every OS: a text format, a + # ZIP-based OOXML format, a binary format with its own extractor, and a + # spreadsheet. Then list, outline and search, so the whole chain from + # bytes to a citable hit is proven cross-platform. + - name: Document ingestion (Markdown, DOCX, PDF, XLSX) + list/outline/search + shell: bash + run: | + export OPENMIND_DATA_DIR="$(mktemp -d)" + export OPENMIND_MACHINE_DIR="$(mktemp -d)" + python -m openmind.cli document --help > /dev/null + WS=$(python -m openmind.cli init --name docsmoke --json \ + | python -c "import json,sys; print(json.load(sys.stdin)['workspace_id'])") + OM_WS="$WS" python - <<'PY' + import json, os, subprocess, sys + ws = os.environ["OM_WS"] + def cli(*args): + out = subprocess.check_output( + [sys.executable, "-m", "openmind.cli", *args, "--json"]) + return json.loads(out) + + expected = { + "sample-requirements.md": ("markdown", "parsed"), + "sample-requirements.docx": ("docx", "parsed"), + "sample-design.pdf": ("pdf", "parsed"), + "sample-tests.xlsx": ("xlsx", "parsed"), + "sample-scanned.pdf": ("pdf", "needs-ocr"), + } + revision = None + for name, (parser, status) in expected.items(): + result = cli("document", "add", "--workspace", ws, + "--path", f"./fixtures/documents/{name}", "--wait") + report = result["import_report"] + assert report["parser"] == parser, (name, report["parser"]) + assert report["parse_status"] == status, (name, report) + if name == "sample-requirements.md": + revision = report["revision_id"] + assert report["segments_created"] > 0, report + assert report["blocks_indexed"] > 0, report + # An image-only PDF must be detected, and must NOT claim OCR ran. + if status == "needs-ocr": + assert report["revision_created"] is False, report + assert not report.get("blocks_indexed"), report + + listing = cli("document", "list", "--workspace", ws) + assert listing["total"] >= 4, listing + parsers = {d["document"]["parser_name"] for d in listing["documents"]} + assert {"markdown", "docx", "pdf", "xlsx"} <= parsers, parsers + + outline = cli("document", "outline", "--workspace", ws, + "--revision", revision, "--limit", "5") + assert outline["count"] == 5 and outline["total"] > 5, outline + assert all(e["evidence_id"] for e in outline["outline"]), outline + + hits = cli("document", "search", "--workspace", ws, + "--query", "REQ-NC-017") + assert hits["count"] > 0, hits + assert hits["query_mode"] == "exact_token", hits + assert all(h["evidence_id"] for h in hits["hits"]), hits + # The locator must be portable: no absolute path may appear. + assert not any(h["locator"].get("document", "").startswith("/") + for h in hits["hits"]), hits + print("document smoke OK:", listing["total"], "documents,", + hits["count"], "hits for REQ-NC-017") + PY + - name: Skill bridge startup shell: bash run: | diff --git a/README.md b/README.md index 26f7fd7..04eddca 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ Point Open Mind at a local repository and it builds persisted artifacts: |---|---| | **Source-traceable knowledge index** | Stores repo-relative paths, content hashes, source locations, file metadata, and search chunks. | | **Canonical Asset model** (v2 Phase 2) | Every indexed file is an **Asset**; every observed version an immutable **Revision**; each revision divided into deterministic **Segments**, each with source-locatable **Evidence**. Historical content is snapshotted in an immutable SHA-256 content store, so evidence for an old revision stays readable after the file changes. Unchanged re-ingestion creates no revision and never re-embeds; removed files are marked removed without erasing history. | +| **Enterprise documents** (v2 Phase 3) | Markdown, text/RST/AsciiDoc, HTML, CSV, DOCX, text-based PDF, XLSX, OpenAPI, JSON Schema and SQL DDL become Assets with the same Revision/Segment/Evidence discipline. Parsing is deterministic and model-free; each block's exact text is snapshotted so a citation survives a parser upgrade. Documents can be appended at any time from anywhere on the machine, live in their own vector collection (the code `search` contract is untouched), and are never merged just because two filenames match. Image-only PDFs are *detected* (`needs-ocr`) — OCR is not performed and never claimed. | | **Verbatim glossary** | Extracts terms/acronyms from definition tables, definition lines, acronym expansions, and code comments; definitions are copied verbatim and carry `source_file`, `line_number`, and `content_hash`. The interactive learn path scans code/config sources; the `.openmind` export walk additionally scans README/docs/GLOSSARY files (primary definition sources). | | **Structure and graph map** | Builds a module tree, per-file definition index, import/dependency graph, entry points, and a name-based call/usage graph. Ambiguous call edges are flagged instead of guessed. | | **Exact-token + hybrid search** | Bare identifiers use token-boundary matching; natural-language queries use vector + lexical retrieval. | @@ -137,11 +138,69 @@ Workspace the project (id p_*; the REST API still says /pr and read-only MCP tools (`list_assets`, `get_asset`, `get_asset_revisions`, `get_evidence`) usable straight from Claude Code. -Deliberately **not** in this phase (still later v2 work): PDF/DOCX/XLSX parsing, -requirement and business-rule extraction, Claim/Relation tables, Knowledge-Graph -edges and requirement-to-code traceability. Nothing here claims document -knowledge or traceability exists yet. Full design: -[docs/v2/phase-2-asset-model.md](docs/v2/phase-2-asset-model.md). +Full design: [docs/v2/phase-2-asset-model.md](docs/v2/phase-2-asset-model.md). + +--- + +## Enterprise Documents (v2 Phase 3) + +Documents are first-class Assets. A requirements DOCX, a design PDF, a test-case +spreadsheet or an OpenAPI description gets the same treatment a source file +does: immutable Revisions, structural Segments, and Evidence you can cite. + +**Supported formats.** Markdown · plain text / RST / AsciiDoc · HTML · CSV/TSV · +DOCX · text-based PDF · XLSX · OpenAPI (JSON/YAML) · JSON Schema · SQL DDL. +Parsing is deterministic and model-free — no LLM reads your documents. + +```bash +# append a document from anywhere on this machine +python -m openmind.cli document add \ + --workspace p_... --path ./Requirements_v3.docx --wait --json + +# find an exact requirement id, with a citable evidence id per hit +python -m openmind.cli document search --workspace p_... --query REQ-NC-017 --json + +# code and documents together, reported separately +python -m openmind.cli knowledge search --workspace p_... --query "NameCheck timeout" --json +``` + +- **Append at any time, from anywhere.** A workspace can start with no + documents. A document does not have to live under a registered source root — + and its absolute origin path never enters the portable database. The import + job carries a content hash and a filename, nothing else. +- **Two documents are never merged because their filenames match.** Re-adding + identical bytes is a `duplicate` (no job, no revision, no vector duplicate). + A filename collision with *different* content is a `possible_revision`: it + writes **nothing** and hands back the exact commands that resolve it. +- **Structure is preserved, and so is what was dropped.** Heading hierarchy, + tables, spreadsheet ranges, page numbers and JSON Pointers all become + locators. An embedded image, a macro, a hidden sheet or an unparseable vendor + SQL statement is *recorded* as unsupported content with a count — never + silently ignored. +- **Nothing is silently truncated.** Every resource limit produces a `partial` + parse plus a warning naming the exact limit and the observed value. +- **Evidence survives the source.** Each block's exact text is snapshotted as + its own content-addressed blob, so a historical citation is recovered + **without re-running a parser** — a newer parser version cannot rewrite + history. An attachment's current-source status is `not-applicable`, not a + false `missing`. +- **Exact identifiers beat similar-looking text.** A search for `REQ-NC-017` + returns blocks that contain it, never `REQ-NC-0170` and never a paragraph that + merely reads like it. Documents live in their own vector collection, so the + existing code `search` behaves exactly as before. +- **Related results are candidates, not relationships.** After an import, + OpenMind reports *observed mentions* — of files, symbols, requirement ids, + API paths, config keys, database objects and glossary terms — each labelled + `status: "candidate"` with a confidence. Nothing is persisted, and no result + claims a document *implements*, *refines*, *verifies* or *contradicts* + anything. + +Deliberately **not** in this phase: Requirement, Business Rule and Design +Decision extraction; Claim/Relation tables; Knowledge-Graph edges; +requirement-to-code traceability; and **OCR** — an image-only PDF is *detected* +and reported `needs-ocr`, with no text invented for it and no claim that OCR +ran. Full design: +[docs/v2/phase-3-document-ingestion.md](docs/v2/phase-3-document-ingestion.md). --- @@ -338,6 +397,11 @@ python -m openmind.cli asset list --workspace --type source-code --json python -m openmind.cli asset revisions --workspace --asset --json python -m openmind.cli asset evidence --workspace --evidence --json +# append an enterprise document (DOCX / PDF / XLSX / Markdown / OpenAPI / ...) +python -m openmind.cli document add --workspace --path ./Requirements.docx --wait +python -m openmind.cli document search --workspace --query REQ-NC-017 --json +python -m openmind.cli knowledge search --workspace --query "review timeout" --json + # expose the knowledge layer to an editor or agent over MCP python -m openmind.cli mcp serve @@ -433,6 +497,27 @@ Implemented MCP tools: | `propose_fix` | Preview a literal find/replace as a unified diff. | | `apply_fix` | Apply the literal replacement only if the test suite stays green. | +Additive read-only tools — the nine above are a frozen contract, and new +capabilities are added *beside* them, never in place of one: + +| Tool | Purpose | +|---|---| +| `list_assets` | A workspace's canonical Assets (bounded). | +| `get_asset` | One Asset plus its current-revision summary. | +| `get_asset_revisions` | An Asset's revision history, newest first. | +| `get_evidence` | One citation with bounded content recovered from the immutable snapshot. | +| `list_documents` | A workspace's document Assets, filterable by parse status or parser. | +| `get_document` | One document with its full parse summary, warnings and unread content. | +| `get_document_outline` | A bounded structural outline of a document revision. | +| `search_documents` | Document search; exact identifiers outrank similar text. | +| `search_knowledge` | Code and documents, returned as separate sections. | +| `find_document_related_candidates` | Observed mentions, labelled `candidate`, never confirmed relations. | + +There is deliberately **no document-write MCP tool**: importing reads a local +file, and exposing that over MCP would let a client make the server read a path +it chose. Claude Code drives `openmind document add` through its shell instead, +where the command is visible. + --- ## Using Open Mind With MCP-Compatible Agents @@ -611,10 +696,16 @@ openmind/ domain/ typed application errors + the types crossing service calls ports/ the three narrow boundaries services depend on services/ use-case orchestration shared by CLI, MCP and FastAPI - (workspace, ingest, job, asset, export, health + the container) + (workspace, ingest, job, asset, document, export, health) content_store.py immutable SHA-256 content-addressed blob store (revision snapshots) segmentation.py deterministic Segment + Evidence drafts (shares boundaries with rag) - migrations/ versioned, checksummed SQLite schema migrations (v0003 = Asset model) + documents/ the document plane: parser SPI + registry, ten deterministic + parsers, safety limits, import planning, commit pipeline and + candidate association. Imports no parser dependency itself. + document_rag.py document retrieval (vector + exact-token + RRF) and the + combined code/document knowledge search + migrations/ versioned, checksummed SQLite schema migrations + (v0003 = Asset model, v0004 = document ingestion) walker.py selection-aware walk, .gitignore handling, hashing detect.py manifest/language detection and stack cues langspec.py declarative language registry @@ -746,6 +837,15 @@ python tests/verify_runtime.py # runtime bootstrap idempotency, worker op python tests/verify_services.py # application services and their typed errors python tests/verify_cli.py # CLI contract: JSON output, exit codes, every command python tests/verify_adapters.py # REST route, MCP tool and skill-bridge compatibility +python tests/verify_content_store.py # immutable blob store: atomic write, reuse, corruption +python tests/verify_asset_model.py # Asset/Revision/Segment/Evidence lifecycle +python tests/verify_document_registry.py # parser SPI: probing, single selection, ambiguity +python tests/verify_document_parsers.py # every format's mandatory cases + determinism +python tests/verify_document_security.py # ZIP bombs, XML entities, no formula/script execution +python tests/verify_document_ingest.py # append, duplicate, collision, removal, evidence +python tests/verify_document_search.py # exact-identifier ranking, candidates never confirmed +python tests/verify_document_cli.py # document CLI: JSON, exit codes, bounds +python tests/verify_document_adapters.py # additive REST/MCP + the compatibility gate ``` The remaining `tests/verify_*.py` files are regression suites (Ask flows, model @@ -764,22 +864,27 @@ against the neutral fixture repos in `fixtures/`. The following are not claimed as complete in the current build: -**v2 enterprise knowledge layer.** Two foundation phases have shipped: +**v2 enterprise knowledge layer.** Three phases have shipped: Phase 1 is the tool-first runtime -([docs/v2/phase-1-core-foundation.md](docs/v2/phase-1-core-foundation.md)), and +([docs/v2/phase-1-core-foundation.md](docs/v2/phase-1-core-foundation.md)), Phase 2 is the canonical **Asset / Revision / Segment / Evidence** model plus the immutable content store -([docs/v2/phase-2-asset-model.md](docs/v2/phase-2-asset-model.md)). The following -later-phase items are **not** implemented; the foundation creates extension -points for them rather than building them: - +([docs/v2/phase-2-asset-model.md](docs/v2/phase-2-asset-model.md)), and Phase 3 +is the deterministic **document-ingestion** plane +([docs/v2/phase-3-document-ingestion.md](docs/v2/phase-3-document-ingestion.md)). +The following later-phase items are **not** implemented; the foundation creates +extension points for them rather than building them: + +- Requirement, Business Rule, Design Decision and Acceptance Criterion + extraction; semantic document classification; document-authority inference; - Claim and Relation models; - the engineering Knowledge Graph and requirement-to-code traceability; -- PDF, DOCX and XLSX parsing; OCR; COBOL and JCL support; requirement and - business-rule extraction; +- **OCR** — image-only PDFs are *detected* and marked `needs-ocr`, never read; +- COBOL, JCL, PPTX and email-archive parsing; Jira and Confluence connectors; - cloud model providers (OpenAI, Anthropic, Bedrock, Azure, Vertex) — the runtime remains local-first with no cloud credentials; -- requirement-to-code traceability, conflict detection and branch overlays; +- conflict detection and branch overlays; +- historical (non-current-revision) document search; - webhook integration and a Bundle 2.0 artifact schema (export stays at schema 1.x); - a typed worker pool or job DAG replacing the current single-worker engine. diff --git a/docs/cli.md b/docs/cli.md index e028a6c..c4941bc 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -226,6 +226,111 @@ segment/evidence id is a typed not-found (exit `1`), never a leak. `assets_removed`, `revisions`, `segments`, `evidence`) — additive; every prior `status` key is unchanged. +### `document` + +Append and query **enterprise documents** (OpenMind v2 Phase 3). A document +becomes an Asset like any other: immutable Revisions, structural Segments, and +Evidence you can cite. See +[docs/v2/phase-3-document-ingestion.md](v2/phase-3-document-ingestion.md). + +Supported formats: Markdown, plain text / RST / AsciiDoc, HTML, CSV/TSV, DOCX, +text-based PDF, XLSX, OpenAPI (JSON/YAML), JSON Schema and SQL DDL. + +```bash +# append a document from ANYWHERE on this machine +python -m openmind.cli document add \ + --workspace p_... --path ./Requirements_v3.docx --wait --json + +# list document assets (bounded; filter by parse status or parser) +python -m openmind.cli document list --workspace p_... --json + +# one document: asset, current revision, parse summary and warnings +python -m openmind.cli document show --workspace p_... --asset a_... --json + +# a bounded STRUCTURAL outline of a revision (structure, not content) +python -m openmind.cli document outline \ + --workspace p_... --revision r_... --limit 500 --json + +# document search (exact identifiers outrank merely similar text) +python -m openmind.cli document search \ + --workspace p_... --query "REQ-NC-017" --json + +# deterministic candidate associations for one document +python -m openmind.cli document related --workspace p_... --asset a_... --json + +# code AND documents, reported separately +python -m openmind.cli knowledge search \ + --workspace p_... --query "NameCheck timeout" --json +``` + +**How to append a document.** `document add` reads a local file, snapshots its +exact bytes into the immutable content store, and enqueues a `document_ingest` +job. **The absolute path never enters the portable database** — the job payload +carries a content hash and a filename. The file does not have to live under a +registered source root; a document that does not is an *attachment*, and its +snapshot is its canonical source. + +**Duplicate and revision decisions.** `document add` reports which of five +things happened, and refuses to guess: + +| `status` | What it means | What was written | +| --- | --- | --- | +| `new_asset` | nothing matched | a new Asset + Revision | +| `revision` | the target Asset exists and the bytes differ | the next Revision | +| `duplicate` | these exact bytes are already a document Revision here | **nothing** | +| `possible_revision` | the filename collides with a DIFFERENT document | **nothing** | +| `unsupported` | no parser claims these bytes | an `unsupported` Asset | + +`possible_revision` exits **1** and hands back the commands that resolve it — +two teams' `requirements.docx` are not the same document, and merging them +because their names match is not recoverable: + +``` +--asset a_... # these bytes are a new revision of that document +--new-asset # this is a different document +--logical-key documents/... # name it yourself +``` + +`--asset`, `--logical-key` and `--new-asset` each name a *different* target for +the same bytes and are mutually exclusive (exit `2`). `--new-asset` produces a +readable deterministic key, e.g. `documents/Requirements_v3--9f2c1a3b.docx`. +`--dry-run` reports the plan and stores nothing. `--version-label` sets an +explicit label; otherwise a label is taken only from a *documented* metadata +field (OpenAPI `info.version`, the DOCX core `revision`) — never guessed from +prose or a filename. + +**How document Evidence is preserved.** Every stored block's exact text is +snapshotted as its own content-addressed blob, so `asset evidence` recovers a +historical citation **without re-running a parser** — a newer parser version +can never rewrite history. For a workspace document the current-source status is +`matches` / `changed` / `missing`; for an attachment it is `not-applicable`, +because its origin path is deliberately not tracked and `missing` would be a +false alarm. + +**Why `related` results are only candidates.** `document related` returns +**observed mentions**, each labelled `status: "candidate"` with a confidence: +`high` for an exact explicit identifier (a file path, a code symbol, a shared +requirement id), `medium` for an exact match after normalization (an API path, a +config key, a database object), `low` for embedding similarity alone. Nothing is +persisted, and no result claims the document *implements*, *refines*, *verifies* +or *contradicts* anything — that needs verification OpenMind does not yet +perform. Likewise `knowledge search` returns code and documents in separate +sections and never asserts a relationship between them. + +**Why OCR is not implemented.** An image-only PDF is *detected* and reported as +`needs-ocr`, and produces no blocks. No text is invented for it, and no result +ever claims OCR ran. Running OCR is Phase 4+. + +**Why Requirement extraction is Phase 4.** This phase ingests and normalizes the +raw document layer. Extracting Requirements, Business Rules, Design Decisions or +Acceptance Criteria — and asserting how they relate to code — requires semantic +verification, and asserting it without that would put unverifiable claims into +immutable history. + +**Why `.openmind` stays at schema 1.1.0.** The artifact bundle is a frozen +integration contract that external consumers depend on. The document model is +not exported through it, so nothing downstream changes. + ### `export` ```bash @@ -319,6 +424,27 @@ esac | `OPENMIND_SOURCELINK_EGRESS` | `0` disables on-demand GitHub source fetching | | `OPENMIND_INGEST_FREE_GPU` | `0` keeps a resident model loaded during bulk embedding | +Document parsing runs inside an explicit resource envelope. Every limit below is +overridable, and hitting one produces a `partial` parse with a warning naming +that exact limit — never a silent truncation. + +| Variable | Default | Bounds | +| --- | --- | --- | +| `OPENMIND_DOCUMENT_MAX_BYTES` | 25,000,000 | whole-document size (refused above it) | +| `OPENMIND_DOCUMENT_MAX_BLOCKS` | 20,000 | blocks per document | +| `OPENMIND_DOCUMENT_MAX_BLOCK_CHARS` | 20,000 | characters per block | +| `OPENMIND_PDF_MAX_PAGES` | 2,000 | pages read from a PDF | +| `OPENMIND_DOCX_MAX_PARAGRAPHS` | 50,000 | DOCX paragraphs | +| `OPENMIND_DOCX_MAX_TABLES` | 2,000 | DOCX tables | +| `OPENMIND_XLSX_MAX_SHEETS` | 100 | worksheets read | +| `OPENMIND_XLSX_MAX_ROWS_PER_SHEET` | 20,000 | rows per sheet | +| `OPENMIND_XLSX_MAX_CELLS` | 500,000 | cells per workbook | +| `OPENMIND_CSV_MAX_ROWS` | 50,000 | CSV/TSV rows | +| `OPENMIND_ZIP_MAX_MEMBERS` | 2,000 | members in a DOCX/XLSX package | +| `OPENMIND_ZIP_MAX_TOTAL_BYTES` | 400,000,000 | total uncompressed size | +| `OPENMIND_ZIP_MAX_MEMBER_BYTES` | 100,000,000 | one member's uncompressed size | +| `OPENMIND_ZIP_MAX_RATIO` | 200 | compression ratio (bomb detection) | + The acceptance runner sets the offline/no-egress set for every test. --- diff --git a/docs/database-migrations.md b/docs/database-migrations.md index d985ba1..e9e87c7 100644 --- a/docs/database-migrations.md +++ b/docs/database-migrations.md @@ -76,6 +76,7 @@ runner switches the connection to explicit-transaction mode | 1 | `baseline` | the schema as it stood before migrations existed: `projects`, `jobs`, `model_config`, `file_index`, `kv`, `ask_history` + its index | | 2 | `paths_sidecar` | moves legacy in-DB project paths into the machine-local sidecar and blanks `projects.paths_json` | | 3 | `asset_model` | the canonical Asset model: `assets`, `asset_revisions`, `segments`, `evidence` + their indexes. Additive — creates only new tables (all `IF NOT EXISTS`), touches no existing row | +| 4 | `document_ingestion` | document ingestion: `segments.content_blob_hash`, `jobs.payload_json`, `document_parses`, `document_index` + their indexes. Additive — two columns with defaults and two new tables | `v0003` adds the OpenMind v2 canonical content-identity model. `assets` references `projects(id)` and the whole subtree cascades on `ON DELETE CASCADE`, @@ -86,15 +87,43 @@ representable. Content bytes live in an immutable content-addressed blob store (`data//objects/…`), never in the database — the DB stores only the SHA-256 hash. See [docs/v2/phase-2-asset-model.md](v2/phase-2-asset-model.md). +`v0004` adds the document-ingestion plane. Two of its four changes deserve a +word about *why*: + +* **`segments.content_blob_hash`** — code Evidence is recovered by slicing + `[startLine, endLine]` out of the revision blob, which is safe forever because + the byte-to-line mapping never changes. A *document* block cannot be recovered + that way: re-deriving it needs a parser rerun, and a future parser version may + legitimately produce different boundaries, which would silently rewrite + history. Document segments therefore snapshot their exact text as their own + content-addressed blob. Existing code segments keep `''` and are deliberately + **not** backfilled. +* **`jobs.payload_json`** — a document import must reach the worker without an + absolute machine path in the portable database. The free `path` column already + means something different per job type, so the safe import payload (staged + blob hash, filename, requested target, import mode) gets its own column. + +`document_parses` is a derived parse projection over an immutable Revision, and +its *presence* for an Asset's current Revision is the definition of "this Asset +is a document" — a recorded fact, never an inference from the file extension. +`document_index` names the vector chunks currently live for an Asset, so a new +current Revision replaces exactly its predecessor's chunks. See +[docs/v2/phase-3-document-ingestion.md](v2/phase-3-document-ingestion.md). + +Unusually, `v0004` declares `upgrade(conn)` rather than `STATEMENTS`: SQLite has +no `ALTER TABLE ... ADD COLUMN IF NOT EXISTS`, and a plain `ADD COLUMN` raises +"duplicate column name" on a second run. Reading `PRAGMA table_info` first keeps +the whole migration idempotent. + ### Upgrading an existing database Nothing to do — open OpenMind and it migrates itself. Concretely: ```text -empty database -> v0001..v0003 create every table -> ledger records 1, 2, 3 -legacy database -> v0001 statements are all no-ops, -> ledger records 1, 2, 3 - v0002/v0003 apply additively (existing data untouched) -current database -> nothing to apply -> no writes +empty database -> v0001..v0004 create every table -> ledger records 1..4 +legacy database -> v0001 statements are all no-ops, -> ledger records 1..4 + v0002..v0004 apply additively (existing data untouched) +current database -> nothing to apply -> no writes ``` A Phase 1 database (already at v0002) upgrades to v0003 with no data loss: @@ -102,6 +131,12 @@ A Phase 1 database (already at v0002) upgrades to v0003 with no data loss: are then backfilled on the next ingestion — without re-embedding unchanged files, reusing their existing Chroma chunks. +A Phase 2 database (at v0003) upgrades to v0004 with no data loss either: +`v0004` adds two columns whose defaults are correct for every existing row, and +two tables that start empty. No project, path, job, Asset, Revision, Segment, +Evidence row, content blob, file-index row, vector collection, map, case, Ask +exchange or template selection is touched. + A legacy database is **baselined, not recreated**. `v0001` is written entirely with `CREATE TABLE IF NOT EXISTS`, so against a database that already has those tables every statement is a no-op and the runner simply records version 1. diff --git a/docs/v2/phase-2-asset-model.md b/docs/v2/phase-2-asset-model.md index 3b18b7b..284768a 100644 --- a/docs/v2/phase-2-asset-model.md +++ b/docs/v2/phase-2-asset-model.md @@ -364,6 +364,15 @@ green; delete stays responsive and the startup janitor race fix is preserved. ## 13. Deferred Phase 3+ work +> **Status update.** Phase 3 has since delivered the document-ingestion half of +> this list — DOCX/PDF/XLSX/Markdown/HTML/CSV/OpenAPI/JSON-Schema/SQL parsing, +> document Assets with block-level Evidence, document search and deterministic +> candidate association. See +> [phase-3-document-ingestion.md](phase-3-document-ingestion.md). Everything +> else below is still deferred, including OCR and requirement traceability. +> This section is left as written to record what Phase 2 itself did and did not +> do. + Intentionally **not** implemented here: PDF/DOCX/XLSX parsing, OCR pipelines, COBOL/JCL parsers, requirement/business-rule extraction, cloud LLM providers, induced Project Lens, Claim/Relation tables, Knowledge-Graph edges, diff --git a/docs/v2/phase-3-document-ingestion.md b/docs/v2/phase-3-document-ingestion.md new file mode 100644 index 0000000..7083a48 --- /dev/null +++ b/docs/v2/phase-3-document-ingestion.md @@ -0,0 +1,719 @@ +# OpenMind v2 — Phase 3: Enterprise Document Ingestion and Appendable Knowledge + +Status: implemented on branch `feat/v2-phase3-document-ingestion`. +Runtime version introduced by this phase: **`1.3.0-dev`**. +Artifact export contract: **unchanged** — `.openmind` schema stays `1.1.0`. + +Phase 3 makes enterprise documents first-class OpenMind Assets. It adds a +deterministic, model-free document-ingestion plane on top of the Phase 2 +Asset/Revision/Segment/Evidence foundation, and it deliberately implements +**none** of the semantic layer above it — no Requirement extraction, no Business +Rules, no Design Decisions, no Claim/Relation tables, no Knowledge-Graph edges, +no OCR, no cloud model calls. See [§16 Deferred work](#16-deferred-phase-4-work). + +--- + +## 1. Baseline: what already worked + +Phase 3 started from a clean working tree at `3a5f6e9` (branch `main`), with the +whole core acceptance suite green. Recorded before any change was made +(`python scripts/run_acceptance.py --json`): + +``` +ok=true — 28 passed, 0 failed, 0 skipped +``` + +| Script | Result | Script | Result | +| --- | --- | --- | --- | +| verify_migrations | 64 passed | verify_facets | 22/22 | +| verify_content_store | 23 passed | verify_guide | 19/19 | +| verify_structure | 14 passed | verify | 17/17 | +| verify_glossary | 20 passed | verify_fixes | 7/7 | +| verify_router | 23 passed | verify_fixes2 | 7/7 | +| verify_diagrams | 7 passed | verify_resources | 15 passed | +| verify_grounding | 8 passed | verify_ask | 15/15 | +| verify_artifacts | 31 passed | verify_ask2 | 29/29 | +| verify_runtime | 31 passed | verify_async_delete | 6/6 | +| verify_services | 97 passed | verify_delete_race | 26/26 | +| verify_cli | 112 passed | verify_source_link | 25 passed | +| verify_adapters | 88 passed | verify_modelserver | 10/10 | +| verify_asset_model | 75 passed | verify_templates | 21/21 | +| verify_asset_cli | 34 passed | verify_asset_adapters | 31 passed | + +Schema head before this phase: **version 3** (`v0001_baseline`, +`v0002_paths_sidecar`, `v0003_asset_model`). +MCP tool set before this phase: **13** — the nine core tools +(`search`, `route`, `dispatch`, `get_glossary`, `find_similar_cases`, +`save_case`, `get_doc`, `propose_fix`, `apply_fix`) plus the four Phase 2 Asset +tools (`list_assets`, `get_asset`, `get_asset_revisions`, `get_evidence`). + +--- + +## 2. Phase 2 Asset model (what this phase builds on) + +``` +Workspace (= the existing project row; id p_*) +└── Asset one logical engineering object, keyed (workspace_id, logical_key) + └── AssetRevision an immutable observation of that Asset's bytes + ├── Segment a stable structural unit inside one revision + └── Evidence a source-locatable citation for a segment +``` + +Facts Phase 3 relies on and does not change: + +* `assets.logical_key` is workspace-relative and portable; no absolute machine + path is ever stored in the database (the source root lives in the + machine-local sidecar, `openmind/machine.py`). +* `asset_revisions.content_blob_hash` is the SHA-256 of the revision's **exact + bytes**, stored in the immutable content store + (`data//objects//`). +* `db.commit_revision` is the single transactional writer: an Asset's + `current_revision_id` is repointed **last**, so it can never name a revision + whose segments and evidence are not yet committed. +* Unchanged content creates **no** new revision (idempotent, revert-safe), and a + removed-then-reappearing Asset is reactivated rather than duplicated. +* Chroma is a *retrieval projection*, never the source of truth. + +Phase 3 adds a parallel, additive plane. It does not modify `v0003`, does not +change `commit_revision`'s existing behaviour, and does not move code Segments. + +--- + +## 3. Target document-ingestion architecture + +``` +Local files under registered roots Manually attached document + │ │ + └──────────────┬────────────────────────┘ + ▼ + Document Intake Service (openmind/documents/intake.py) + │ read bytes → SHA-256 → immutable staged blob + ▼ + Parser Registry (openmind/documents/registry.py) + │ probe → exactly one parser (or a typed failure) + ▼ + Normalized ParsedDocument (openmind/documents/models.py) + metadata · blocks · warnings · coverage + │ + ▼ + Asset / Revision / Segment / Evidence + document_parses + │ + ┌──────────────┴───────────────┐ + ▼ ▼ + documents_ collection deterministic candidate association + │ │ + └──────────────┬───────────────┘ + ▼ + CLI · REST · MCP · Claude Code +``` + +Raw bytes and normalized content stay on the machine. No project document +content leaves the machine in this phase; no parser dependency performs network +I/O or telemetry. + +--- + +## 4. Parser SPI + +`openmind/documents/` holds the whole plane: + +``` +openmind/documents/ +├── __init__.py lazy re-exports; importing it pulls in NO parser dependency +├── models.py ParsedDocument, DocumentBlock, warnings, probe, context +├── security.py limits, ZIP package safety, safe XML, bounded decoding +├── registry.py register / list_parsers / select / parse (lazy imports) +├── probe.py magic + package-structure detection (never extension alone) +├── text_parser.py plain text, RST, AsciiDoc +├── markdown_parser.py Markdown / CommonMark subset +├── html_parser.py HTML (stdlib HTMLParser, scripts and styles dropped) +├── csv_parser.py CSV / TSV (stdlib csv, bounded sniffing) +├── docx_parser.py DOCX (python-docx) +├── pdf_parser.py PDF (pypdf) +├── spreadsheet_parser.py XLSX (openpyxl) +├── openapi_parser.py OpenAPI 2/3 (JSON + YAML) +├── json_schema_parser.py JSON Schema +├── sql_parser.py SQL DDL (sqlglot, honest text fallback) +├── pipeline.py parse → blobs → segments → transactional commit → vectors +└── candidates.py deterministic candidate association +``` + +### 4.1 Contract + +```python +class DocumentParser(Protocol): + name: str + version: str + def supports(self, probe: DocumentProbe) -> bool: ... + def parse(self, content: bytes, context: DocumentParseContext) -> ParsedDocument: ... +``` + +`DocumentProbe` carries `filename`, `extension`, `declared_media_type`, +`detected_media_type`, `magic` (signature facts: `zip`, `pdf`, `ole2`, `utf8`, +`bom`, `zip_members` for OOXML package structure), `size`, and a bounded `head` +sample of the bytes. **A parser is never selected from the extension alone.** +For ZIP-based OOXML formats the probe verifies package structure +(`word/document.xml` for DOCX, `xl/workbook.xml` for XLSX) before a parser may +claim the file. + +`DocumentParseContext` carries the portable `logical_key`, the original +`filename`, the workspace id, the effective `DocumentLimits`, and free-form +`parser_options`. + +### 4.2 Registry rules + +1. **Deterministic order** — parsers are registered with an explicit integer + priority; ties break on parser name. `list_parsers()` returns that order. +2. **Exactly one parser is selected.** `select(probe)` returns the single + highest-priority parser whose `supports()` is true. +3. **Ambiguity fails clearly.** Two parsers at the *same* priority both claiming + a probe raise `AmbiguousParser` rather than silently picking one. +4. **Missing optional dependency** → `ParsedDocument(status="unsupported")` with + `reason="dependency_unavailable"` and the missing distribution named. Only + that format fails. +5. **Unsupported format** → `status="unsupported"`, never an empty successful + parse. +6. **Parser imports are lazy** — `registry.select()` imports a parser module + only when its probe matches, and the third-party library only inside + `parse()`. Importing `openmind.documents` imports no parser dependency. +7. **Artifact export stays dependency-free**: `openmind export` never imports + `openmind.documents`. + +--- + +## 5. Normalized document model + +``` +ParsedDocument +├── parser_name, parser_version, schema_version ("1.0") +├── status parsed | partial | needs-ocr | encrypted | unsupported | failed +├── title, media_type +├── metadata DocumentMetadata (author/created/modified/producer/version_label/extra) +├── blocks [DocumentBlock] +├── warnings [DocumentParseWarning] (code, message, locator?) +├── unsupported_content [UnsupportedContent] (kind, count, detail) +└── coverage {"blocks": n, "indexable": n, "truncated": bool, ...} +``` + +``` +DocumentBlock +├── block_key stable, deterministic, unique within the document +├── block_type document | section | heading | paragraph | list-item | code-block +│ | table | table-row | sheet | cell-range | page | api-operation +│ | schema-definition | sql-object +├── ordinal dense, 0-based, document order +├── parent_key the containing block's key ("" for the root) +├── heading_path [str] ancestor headings, outermost first +├── text the represented text +├── content_mode verbatim | derived +├── locator a portable document locator (see §6) +├── metadata {} format-specific facts +└── indexable whether this block is embedded into the document collection +``` + +**Content mode.** `verbatim` is used only when `text` is an exact textual +representation of the source. A synthesized table rendering, a spreadsheet row +serialized as `header=value` pairs, an OpenAPI operation summary and the +document root block are `derived`. Markdown/text/HTML paragraphs, DOCX +paragraph runs, PDF page text and code blocks are `verbatim`. + +**Indexable.** Structural containers (the `document` root, an empty `section`, +a `table` wrapper whose rows carry the content) are stored as Segments but not +embedded. Only content-bearing leaves are indexed, so retrieval is not diluted +by empty scaffolding. Every stored block still becomes a Segment with Evidence. + +--- + +## 6. Document source locators + +The Phase 2 `source-range` locator remains valid for code and is untouched. +Document locators are additive and always carry a **portable logical document +key**, never an absolute path. Page and line numbers presented to users are +1-based. + +| Format | `kind` | Fields | +| --- | --- | --- | +| Markdown / text / RST / AsciiDoc | `text-range` | `document`, `startLine`, `endLine`, `headingPath` | +| HTML | `html-element` | `document`, `element`, `elementIndex`, `domPath`, `headingPath` | +| DOCX paragraph | `docx-paragraph` | `document`, `paragraphIndex`, `headingPath` | +| DOCX table row | `docx-table-row` | `document`, `tableIndex`, `rowIndex`, `cellRange` | +| PDF | `pdf-block` | `document`, `page`, `blockIndex` | +| XLSX | `spreadsheet-range` | `document`, `sheet`, `range` | +| CSV | `spreadsheet-range` | `document`, `sheet` (`""`), `range`, `rowIndex` | +| OpenAPI / JSON Schema | `json-pointer` | `document`, `pointer` | +| SQL DDL | `text-range` | `document`, `startLine`, `endLine`, `symbol` | + +--- + +## 7. Segment-content snapshots + +Phase 2 Evidence for **code** is reconstructed by slicing `[startLine, endLine]` +out of the revision blob. That is safe because the mapping from bytes to lines +is fixed forever. + +For a **document** it is not. Re-deriving a DOCX paragraph or a PDF page block +requires re-running a parser, and a future parser version may legitimately +produce different block boundaries. Reconstructing historical Evidence that way +would silently rewrite history. + +So Phase 3 adds an optional **segment content blob**: + +```sql +ALTER TABLE segments ADD COLUMN content_blob_hash TEXT NOT NULL DEFAULT ''; +``` + +For a document segment the pipeline + +1. stores the block's exact represented text (UTF-8) in the content store, +2. records that blob's SHA-256 in `segments.content_blob_hash`, +3. sets `segments.content_hash` to the SHA-256 of that same exact text, +4. sets the Evidence `content_hash` to the same value. + +Historical document Evidence is then recovered **from the block blob**, with no +parser rerun. Existing code segments keep `content_blob_hash = ''` and continue +to resolve through the line-range path; Phase 2 rows are **not** backfilled. + +--- + +## 8. Document parse records + +```sql +CREATE TABLE document_parses ( + revision_id TEXT PRIMARY KEY, + parser_name TEXT NOT NULL, + parser_version TEXT NOT NULL, + schema_version TEXT NOT NULL, + status TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + media_type TEXT NOT NULL DEFAULT '', + metadata_json TEXT NOT NULL DEFAULT '{}', + warnings_json TEXT NOT NULL DEFAULT '[]', + unsupported_json TEXT NOT NULL DEFAULT '[]', + coverage_json TEXT NOT NULL DEFAULT '{}', + structure_hash TEXT NOT NULL, + created_at TEXT NOT NULL, + FOREIGN KEY(revision_id) REFERENCES asset_revisions(id) ON DELETE CASCADE +) +``` + +It is a **derived parse projection over an immutable Revision**. In Phase 3 a +Revision gets exactly one parse result; the parser name and version are +recorded; the output is deterministic (`structure_hash` is the SHA-256 over the +ordered `(block_key, block_type, ordinal, parent_key, content_hash)` tuples, so +a re-parse that changes structure is detectable). Old Revisions are never +automatically re-parsed, and a historical parse is never silently replaced by a +different parser version. A future analysis-generation model may allow that. + +**The presence of a `document_parses` row for an Asset's current Revision is the +definition of "this Asset is a document."** That is a recorded fact, not an +inference. + +--- + +## 9. Document vector projection + +Documents get their **own collection per workspace** so the existing `search` +contract cannot change: + +``` +code_ unchanged — code/config chunks (existing `search`) +cases_ unchanged +documents_ NEW — document block chunks +``` + +```sql +CREATE TABLE document_index ( + workspace_id TEXT NOT NULL, + asset_id TEXT NOT NULL, + revision_id TEXT NOT NULL, + chunk_ids_json TEXT NOT NULL DEFAULT '[]', + updated_at TEXT NOT NULL, + PRIMARY KEY(workspace_id, asset_id), + FOREIGN KEY(asset_id) REFERENCES assets(id) ON DELETE CASCADE +) +``` + +Each indexed chunk carries: `workspace_id`, `asset_id`, `revision_id`, +`segment_id`, `evidence_id`, `logical_key`, `title`, `asset_type`, `block_type`, +`heading_path`, `parser_name`, `page`, `sheet`, `json_pointer`, `content_hash`. +Only relevant fields are populated. + +**Active-revision behaviour.** Document search returns the current active +Revision. When a new Revision becomes current the pipeline removes the previous +active chunks, adds the new ones and updates `document_index`; every historical +Segment and Evidence row is preserved. Historical search may be added later. + +**Lifecycle.** `vectorstore._pid_of` recognizes the `documents_` prefix, so: +workspace terminate drops the document collection, workspace delete drops it, +and the startup orphan sweep classifies it correctly (a *deleting* workspace is +still "known" and therefore not an orphan). Drains stay batched, interruptible +and protected by the existing in-flight guard — the delete-race fixes are not +touched. + +Chunk ids are stable and content-derived +(`d_` + `sha1(asset_id|revision_id|block_key)[:18]`), so a retried upsert after +a mid-commit failure writes the same ids and is idempotent. + +--- + +## 10. Format support + +| Format | Parser | Support | Not supported in Phase 3 | Security limits | +| --- | --- | --- | --- | --- | +| Markdown | `markdown` | headings + hierarchy, paragraphs, lists, fenced code, block quotes, pipe tables, line locators | reference/footnote resolution, inline HTML rendering, MDX | `DOCUMENT_MAX_BYTES`, `DOCUMENT_MAX_BLOCKS`, `DOCUMENT_MAX_BLOCK_CHARS` | +| Plain text / RST / AsciiDoc | `text` | line ranges, blank-line paragraphs, underline/`=`/`#` headings, list items, indented code blocks | RST/AsciiDoc directives, includes, substitutions (reported as unsupported content) | as above | +| HTML | `html` | ``, headings, paragraphs, list items, tables, `pre`/`code`; scripts, styles, comments and `hidden` content dropped | JavaScript execution, external resource fetch, CSS-computed visibility | as above; no network | +| DOCX | `docx` | core properties, paragraphs, heading styles + hierarchy, list items, tables/rows/cells, code-like styles, section order | macros, embedded images (recorded as unsupported content), tracked changes, headers/footers | ZIP package caps, safe XML, `DOCX_MAX_PARAGRAPHS`, `DOCX_MAX_TABLES` | +| PDF (text) | `pdf` | metadata, per-page text, block ordering where available, 1-based page locators | OCR, table recovery, form fields, annotations | `PDF_MAX_PAGES`, `DOCUMENT_MAX_BYTES` | +| XLSX | `xlsx` | workbook props, visible sheets, bounded used ranges, row/cell values, formulas **as text**, merged-cell metadata, header detection | formula evaluation, external links, macros (`.xlsm` unsupported), charts | ZIP package caps, `XLSX_MAX_SHEETS`, `XLSX_MAX_ROWS_PER_SHEET`, `XLSX_MAX_CELLS` | +| CSV / TSV | `csv` | encoding fallback, bounded dialect sniffing, header row, row locators | nested/multi-table files, type inference | `CSV_MAX_ROWS`, `DOCUMENT_MAX_BLOCK_CHARS` | +| OpenAPI (JSON/YAML) | `openapi` | `info.title`/`info.version`, paths + operations, parameters, request bodies, responses, component schemas, JSON-Pointer locators | remote `$ref` resolution, business-meaning inference, validation | safe YAML (`yaml.safe_load`), `DOCUMENT_MAX_BLOCKS` | +| JSON Schema | `json-schema` | root, `definitions`/`$defs`, properties, constraints, internal `$ref` | remote `$ref` fetch, schema validation | as above; **no network** | +| SQL DDL | `sql` | tables, views, columns, indexes, constraints, sequences, line ranges; honest bounded-text fallback on unsupported dialect syntax | dialect-perfect parsing, execution | `DOCUMENT_MAX_BLOCKS` | + +**Truncation is never silent.** When a limit is reached the parser keeps the +already-extracted content, sets `status="partial"`, appends a warning naming the +exact limit and the observed value, and reports `coverage`. + +**Honest PDF statuses.** An image-only page produces no fabricated text. A PDF +with no meaningful extractable text becomes `needs-ocr`; an encrypted PDF that +cannot be opened with the empty password becomes `encrypted`. **No OCR is +performed in Phase 3** and no result ever claims OCR was performed. + +--- + +## 11. Security and resource controls + +Document parsing handles untrusted files, so the controls are explicit and +tested (`tests/verify_document_security.py`). + +**ZIP package safety** (`security.inspect_zip`) — for DOCX/XLSX: +member-count cap, total-uncompressed-size cap, per-member size cap, path +traversal rejection (`..`, absolute, drive-letter, backslash members), +compression-ratio (zip-bomb) detection. Members are read **in memory from the +already-validated archive**; nothing is ever extracted to a filesystem path, and +no embedded content is executed. + +**XML safety** — `defusedxml` is installed as a dependency and +`defusedxml.defuse_stdlib()` is applied before any OOXML parse, so external +entities, external DTDs and entity-expansion bombs are refused by the underlying +stdlib parsers that `python-docx`/`openpyxl` use. When `defusedxml` is absent the +DOCX/XLSX parsers report `dependency_unavailable` rather than parsing unsafely. + +**Configurable limits** (`openmind/config.py`, each overridable by an +`OPENMIND_*` environment variable): + +``` +DOCUMENT_MAX_BYTES 25_000_000 +PDF_MAX_PAGES 2_000 +DOCX_MAX_PARAGRAPHS 50_000 +DOCX_MAX_TABLES 2_000 +XLSX_MAX_SHEETS 100 +XLSX_MAX_ROWS_PER_SHEET 20_000 +XLSX_MAX_CELLS 500_000 +CSV_MAX_ROWS 50_000 +DOCUMENT_MAX_BLOCKS 20_000 +DOCUMENT_MAX_BLOCK_CHARS 20_000 +ZIP_MAX_MEMBERS 2_000 +ZIP_MAX_TOTAL_BYTES 400_000_000 +ZIP_MAX_MEMBER_BYTES 100_000_000 +ZIP_MAX_RATIO 200 +``` + +**Parsing isolation.** `registry.parse()` catches every parser exception and +returns `status="failed"` with the exception type and message. A parser failure +fails that document's job step, never the worker process, and never leaves a +partial current Revision. + +--- + +## 12. Append-document workflow and identity rules + +`DocumentService` (`runtime.documents`, `ServiceContainer.documents`) exposes +`plan_import`, `add_document`, `get_document`, `list_documents`, `get_outline`, +`search`, `search_knowledge`, `find_related_candidates`. + +``` +Read local bytes → SHA-256 → immutable staged blob (content store) + → persist a safe import payload (jobs.payload_json — NO absolute path) + → enqueue a `document_ingest` job + → parse from the staged blob + → Asset / Revision / Segments / Evidence + document_parses + → document vector projection + → deterministic candidate association + → import report +``` + +The job payload may contain only `staged_blob_hash`, `original_filename`, +`requested_asset_id`, `requested_logical_key`, `import_mode`, `version_label`, +`parser_options`. **The absolute origin path never enters the portable +database.** + +### 12.1 Identity + +| Origin | Logical key | +| --- | --- | +| Under a registered source root | `workspace_id` + workspace-relative path (unchanged Phase 2 rule) | +| Manually attached | `documents/<normalized-filename>` | +| `--logical-key K` | `K` (normalized) | +| `--new-asset` | `documents/<stem>--<content-hash-prefix>.<ext>` — readable and deterministic, never opaque-random | + +Two different documents are **never** silently merged because their filenames +match. + +### 12.2 Import decisions + +| Condition | `status` | Effect | +| --- | --- | --- | +| Content hash already exists as a document Revision in the workspace | `duplicate` | Returns the existing Asset + Revision. **No job, no Revision, no vector duplicate.** | +| `--asset A` supplied, content differs | `revision` | Next Revision of `A` (validated: belongs to the workspace, is document-compatible) | +| `--asset A` supplied, content unchanged | `duplicate` | As above | +| `--logical-key K` | `new_asset` / `revision` | Find or create that Asset | +| Default key exists with **different** content and no explicit choice | `possible_revision` | **Nothing is written.** Returns the candidate Asset, its current Revision, both content hashes and the exact retry commands. | +| `--new-asset` | `new_asset` | Distinct deterministic key | +| Nothing matches | `new_asset` | Create | + +### 12.3 Version label + +Only explicit deterministic metadata is used: OpenAPI `info.version`, the DOCX +core `revision` property, a clearly-defined PDF metadata version field, or a +caller-supplied `--version-label` (which always wins). Versions are **never** +inferred from prose or filename patterns. `revision.status` stays `unknown` — +approval authority is not inferred. + +### 12.4 Workspace discovery vs. code discovery + +`config` now separates the policies: + +``` +CODE_INDEX_EXTENSIONS = INDEX_EXTENSIONS (unchanged: .java .ts .yaml .sql .html …) +TEXT_DOCUMENT_EXTENSIONS = .md .markdown .rst .adoc .asciidoc .txt +BINARY_DOCUMENT_EXTENSIONS = .docx .pdf .xlsx +STRUCTURED_DOCUMENT_EXTENSIONS = .csv .tsv +DOCUMENT_DISCOVERY_EXTENSIONS = (the three above) − CODE_INDEX_EXTENSIONS +``` + +The subtraction is load-bearing: a file already owned by the **code** pipeline +(`.html`, `.yaml`, `.json`, `.sql`, …) is never also discovered as a workspace +document, so nothing is ingested twice. Those formats are still fully parseable +**on demand** through `document add`, which is how an OpenAPI YAML or a schema +SQL file becomes a document Asset. And no document extension is added to +`INDEX_EXTENSIONS`, so a `.pdf` can never reach `walker.read_text`. + +A **full** ingest discovers code assets, discovers document assets, runs both +pipelines, and marks removed assets in both. A **filtered** code ingest never +prunes document Assets, and a filtered document ingest never rebuilds or prunes +code Assets. + +**Removal.** A workspace document that disappears has its active document chunks +removed and its Asset marked `removed`; every Revision, Segment, Evidence row +and content blob is preserved. A **manually attached** snapshot never becomes +`removed` because its external origin path vanished — the snapshot is the +canonical source (`source_kind="attachment"`). + +--- + +## 13. Evidence retrieval + +`AssetService.get_evidence` keeps its exact Phase 2 behaviour for code Evidence +(source-range slice out of the revision blob + live-file comparison). Document +Evidence is resolved through an additive locator-resolver path: + +1. If the Segment has a `content_blob_hash`, the exact block text is read from + that blob and verified against the Evidence `content_hash` → `snapshot: + available` (or `corrupt` on mismatch). **No parser is rerun.** +2. `current_source.status` is: + * `not-applicable` for an attachment (its origin path is deliberately not + tracked, so `missing` would be a false claim); + * `matches` / `changed` / `missing` for a workspace document file, decided by + comparing the **whole current document's** content hash against the + Revision's — block-level re-resolution is not deterministic across parser + versions, so document-level comparison is the honest answer. + +The response adds `parser`, `revision`, `snapshot`, `current_source` and +`truncated`; bounded output truncates honestly. + +--- + +## 14. Retrieval + +### 14.1 Document search (`openmind/document_rag.py`) + +`vector retrieval + lexical retrieval + exact identifier matching`, fused with +**RRF**, bounded. An exact Requirement ID, API path, error code or identifier is +matched on token boundaries and is **promoted above** any merely +embedding-similar result — the same guarantee the code RAG already gives. + +Each hit carries `asset_id`, `revision_id`, `segment_id`, `evidence_id`, +`title`, `logical_key`, `block_type`, `heading_path`, `locator`, `excerpt`, +`score`, `retrieval_sources`. Filters: `asset_type`, `parser`, `block_type`, +`logical_key`. Removed Assets are excluded by default. + +### 14.2 Combined knowledge search + +`search_knowledge` returns code and documents **separately**: + +```json +{"query": "...", + "code": {"hits": []}, + "documents": {"hits": []}, + "grounding": {"codeCount": 0, "documentCount": 0}} +``` + +It never claims a document hit *implements* or *refines* a code hit. This is +retrieval, not relationship inference. The existing MCP `search` tool stays +code-oriented and byte-for-byte unchanged. + +### 14.3 Deterministic candidate association + +`find_related_candidates` compares an imported document against existing +knowledge using deterministic signals first: + +| # | Signal | Candidate type | Confidence | +| --- | --- | --- | --- | +| 1 | exact workspace file path | `mentions-file` | high | +| 2 | exact code symbol (Segment `symbol`) | `mentions-symbol` | high | +| 3 | Requirement-like identifier (`REQ-NC-017`) | `mentions-document` / `similar-content` | high | +| 4 | change-request / ticket identifier (`ABC-1234`) | `mentions-document` | high | +| 5 | API path + HTTP method | `mentions-api` | medium | +| 6 | error code | `mentions-configuration` | medium | +| 7 | configuration key (dotted) | `mentions-configuration` | medium | +| 8 | database object | `mentions-database-object` | medium | +| 9 | message / topic name | `mentions-configuration` | medium | +| 10 | exact glossary term | `mentions-document` | high | +| 11 | semantic retrieval fallback | `similar-content` / `possibly-related` | low | + +Every candidate carries `candidate_type`, `confidence`, `reason`, +`document_evidence`, `target`, `target_evidence`, `retrieval_method` and +`status: "candidate"`. `implements`, `refines`, `verifies` and `contradicts` are +**never** returned — they require the semantic verification deferred to Phase 4. +**Candidates are computed on demand and are never persisted as canonical +Relations.** + +--- + +## 15. Compatibility boundaries + +| Surface | Guarantee | +| --- | --- | +| Runtime | CLI, MCP, FastAPI and tests keep using the same `OpenMindRuntime`; `documents` is an additive service property. | +| REST | No route removed or renamed. The public API keeps saying `/projects`, never `/workspaces`. All Phase 3 routes are additive. `/ocr` stays separate — a PDF import is **never** silently routed through OCR. | +| MCP | The nine core tools and the four Phase 2 Asset tools are unchanged. Six read-only document tools are added; **no document-write MCP tool exists in Phase 3** (Claude Code drives imports through the CLI). MCP startup still does not start the worker. | +| Artifact | `.openmind schemaVersion` stays `1.1.0`; the document model is **not** exported through Bundle 1.x. | +| Skill bridge | JSON-lines protocol unchanged, still independent of the application database. | +| Job engine | The single worker is not replaced. `document_ingest` is an additional job type; `jobs.payload_json` is an additive nullable column. | +| Databases | A Phase 1 or Phase 2 database migrates to head with projects, paths, jobs, Asset history, content blobs, file index, vector collections, maps, cases, Ask history and template metadata intact — `v0004` only `CREATE`s tables/indexes and `ADD COLUMN`s with defaults. | + +### 15.1 Schema (migration `v0004_document_ingestion`) + +* `ALTER TABLE segments ADD COLUMN content_blob_hash TEXT NOT NULL DEFAULT ''` +* `ALTER TABLE jobs ADD COLUMN payload_json TEXT NOT NULL DEFAULT '{}'` +* `CREATE TABLE document_parses (…)` +* `CREATE TABLE document_index (…)` +* indexes: `idx_document_parses_status`, `idx_document_index_ws`, + `idx_segments_blob` + +The migration is written so it is safe to re-run against a database that already +has the column (SQLite has no `ADD COLUMN IF NOT EXISTS`), via an `upgrade(conn)` +function that inspects `PRAGMA table_info` first. + +### 15.2 Dependency policy + +| Package | License | Why | Used by | +| --- | --- | --- | --- | +| `python-docx` | MIT | DOCX paragraphs/tables/styles | `docx_parser` | +| `pypdf` | BSD-3-Clause | pure-Python PDF text extraction | `pdf_parser` | +| `openpyxl` | MIT | XLSX cells, formulas-as-text, merges | `spreadsheet_parser` | +| `defusedxml` | PSF-2.0 | hardens the stdlib XML parsers OOXML uses | `security` | + +All four are permissive and compatible with this MIT project. **No AGPL parser +dependency is introduced** — in particular PyMuPDF is deliberately *not* used +for PDF text extraction. None of the four performs telemetry or network calls. +All are imported lazily, a missing one fails only its format, and the +dependency-free `.openmind` artifact export CI job stays green. + +--- + +## 16. Testing strategy + +New acceptance scripts, all registered in `scripts/run_acceptance.py` (an +unregistered `tests/verify_*.py` already fails the runner, so a missing core +document test fails the manifest): + +| Script | Covers | +| --- | --- | +| `verify_document_registry` | probe facts, deterministic order, single selection, ambiguity failure, `dependency_unavailable`, `unsupported`, lazy imports | +| `verify_document_parsers` | every mandatory per-format case in the task spec, plus byte-for-byte determinism on a repeated parse | +| `verify_document_security` | ZIP traversal, zip bomb, member caps, XML entities disabled, oversized PDF, oversized workbook, formulas not executed, HTML scripts ignored, no remote `$ref` fetch | +| `verify_document_ingest` | the 13 mandatory import cases + the mandatory Evidence cases | +| `verify_document_search` | exact-ID retrieval, exact-outranks-semantic, evidence ids, combined search shape, candidate association, low-confidence labelling, never-confirmed | +| `verify_document_cli` | `document add/list/show/outline/search/related`, `knowledge search`, JSON contract, exit codes, bounds, `--dry-run` | +| `verify_document_adapters` | additive REST routes, cross-workspace 404, the 13 pre-existing MCP tools still present, the 6 new document tools additive | + +Fixtures live in `fixtures/documents/` and are **invented, neutral content** +about a fictional "NameCheck" service. `sample-requirements.docx`, +`sample-design.pdf` and `sample-tests.xlsx` are generated deterministically by +`scripts/build_document_fixtures.py` (documented in the fixture README) and are +each a few kilobytes. + +--- + +## 17. Result + +Implemented as designed. The full suite, every tier: + +``` +python scripts/run_acceptance.py --all --json +ok=true — 36 passed, 0 failed, 0 skipped (1,568 individual checks) +``` + +| Script | Checks | Script | Checks | +| --- | --- | --- | --- | +| verify_document_registry | 42 | verify_document_search | 79 | +| verify_document_parsers | 190 | verify_document_cli | 90 | +| verify_document_security | 74 | verify_document_adapters | 93 | +| verify_document_ingest | 102 | verify_migrations | 76 (was 64) | + +Schema head: **4**. MCP tool set: **19** (9 core + 4 Asset + 6 document), with +the first thirteen unchanged. Artifact contract: **1.1.0**, unchanged. + +Five bugs the suites caught during implementation, all fixed: + +1. `DocumentBuilder.full` was a silent early-exit signal — a parser that broke + out of its loop produced a truncated document still labelled `parsed`. +2. `decode_text` tried UTF-16 before the single-byte fallbacks, and a UTF-16 + decode of arbitrary text succeeds whenever the length is even, silently + turning `caf\xe9 latte` into CJK mojibake. +3. `cmd_document_add` mutated `ok`/`error` *after* emitting, so a + `possible_revision` printed `"ok": true` while exiting non-zero. +4. `extract_terms` fed identifier COMPONENTS back as lexical terms, so + `REQ-NC-017` also matched `REQ-NC-018` and `REQ-NC-019`. +5. A removed-then-reappeared document was reactivated but never re-indexed — + active in the database and invisible to search, which is the worst failure + mode because nothing looks wrong. + +Verified out of band as well: with `python-docx`, `pypdf`, `openpyxl` and +`defusedxml` all blocked, the artifact export still runs, the registry still +loads, Markdown/HTML/CSV still parse, and DOCX/PDF/XLSX report +`dependency_unavailable` for the *right* parser. A database populated by the +Phase 2 build and opened by this one migrates 3 → 4 with zero differences across +projects, jobs, file index, assets, revisions, segments, evidence, Ask history, +kv and project meta. + +--- + +## 18. Deferred (Phase 4+) work + +Explicitly **not** implemented here, and never described as implemented: + +* Requirement, Business Rule, Design Decision and Acceptance Criterion + extraction; +* semantic document classification and document-authority inference; +* canonical Claim and Relation tables, Knowledge-Graph edges, + Requirement-to-Code traceability, conflict detection, induced Project Lens; +* OCR execution (image-only PDFs are *detected* and marked `needs-ocr` only); +* COBOL / JCL / PPTX / email-archive parsing; Jira and Confluence connectors; +* branch or PR overlays, webhooks, Bundle 2.0, Titan Mind, Neo4j; +* cloud OpenAI / Anthropic / Bedrock / Azure / Vertex calls; +* historical (non-current-revision) document search; +* a worker-pool rewrite, new Agent Skills, and any new UI. diff --git a/fixtures/documents/README.md b/fixtures/documents/README.md new file mode 100644 index 0000000..6f6230a --- /dev/null +++ b/fixtures/documents/README.md @@ -0,0 +1,73 @@ +# Document parser fixtures + +Small, neutral, **invented** documents for the Phase 3 document-ingestion tests. + +Everything here describes a fictional "NameCheck" screening service. No content +is copied from any real, proprietary or employer document, and no identifier, +endpoint, error code or requirement id in these files refers to a real system. + +## Text fixtures (checked in as source) + +These are ordinary text and are edited directly — a readable diff is worth more +than uniformity with the generated ones. + +| File | Exercises | +| --- | --- | +| `sample-requirements.md` | heading hierarchy, paragraphs, list items, a fenced code block, a pipe table, a block quote, exact line locators | +| `sample-design.html` | `<title>`, headings, paragraphs, lists, a table, `pre`/`code`; a `<script>`, a `<style>` and two hidden paragraphs that must **not** be indexed | +| `sample-cases.csv` | header detection, row locators, dialect sniffing | +| `sample-openapi.yaml` | `info.title`/`info.version`, path operations, a path-level parameter, request body, responses, component schemas, JSON-Pointer locators | +| `sample-schema.json` | root schema, `$defs`, properties, constraints, an internal `#/$defs` `$ref` that IS resolved, and a remote `$ref` that must **never** be fetched | +| `sample-schema.sql` | tables, columns, constraints, an index, a view, plus a PL/SQL package body that `sqlglot` cannot structure and that must fall back to bounded verbatim text with an exact line range | + +## Binary fixtures (generated) + +Run: + +```bash +python scripts/build_document_fixtures.py +``` + +| File | Built with | Exercises | +| --- | --- | --- | +| `sample-requirements.docx` | `python-docx` | core properties (including `revision` → version label), heading styles and hierarchy, list styles, a table, and one embedded image recorded as unsupported content | +| `sample-tests.xlsx` | `openpyxl` | workbook properties, two visible sheets and one hidden one, a merged banner cell, bounded used ranges, and two formulas that must be stored **as formulas** and never evaluated | +| `sample-design.pdf` | hand-written PDF bytes | metadata, two pages of extractable text, 1-based page locators | +| `sample-scanned.pdf` | hand-written PDF bytes | a page with a drawn rectangle and **no text operators** → must be reported `needs-ocr`, with no fabricated text | +| `sample-encrypted.pdf` | hand-written PDF bytes | an `/Encrypt` trailer that does not open with the empty password → must be reported `encrypted` | + +### Why the PDFs are hand-written + +Writing the PDF bytes directly keeps a PDF-*writing* dependency out of the +repository (OpenMind only ever *reads* PDFs), keeps each file well under 2 KB, +and puts the bytes fully under our control — which is what makes "this page has +no extractable text" and "this file is encrypted" reliable test inputs instead +of a library's changing idea of them. + +### What regeneration guarantees + +An unchanged document must produce **no new Revision**. A fixture whose *content* +drifted between runs would change its content hash and make that guarantee +untestable. So every timestamp is pinned to a fixed epoch, and the OOXML packages +are rewritten with fixed member order and fixed member timestamps +(`normalize_zip` in the generator) — `openpyxl` in particular overwrites +`dcterms:modified` with the wall clock during `save()`, so that field is +rewritten afterwards. + +The guarantee is **content**-level, and it differs by format for a reason: + +| Format | Reproducible | Why | +| --- | --- | --- | +| `.pdf` | byte for byte | the generator writes the bytes itself; no compressor is involved | +| `.docx`, `.xlsx` | member for member | a DOCX/XLSX is a DEFLATE archive, and the compressed bytes depend on the platform's zlib build. Same parts in, different container bytes out. | + +So on the **same machine**, regenerating produces an empty diff. Regenerating on +a *different platform* can legitimately change the OOXML container bytes while +every part inside is identical — that is not drift, and CI compares the member +map rather than the archive so it is not reported as such. Storing the archives +uncompressed would make them byte-identical everywhere, but it inflates +`sample-requirements.docx` from 37 KB to 832 KB, which is not a trade worth +making for a test fixture. + +Nothing depends on the exact committed bytes: the tests hash the fixture at +runtime, and no expected hash is written down anywhere. diff --git a/fixtures/documents/sample-cases.csv b/fixtures/documents/sample-cases.csv new file mode 100644 index 0000000..1112b76 --- /dev/null +++ b/fixtures/documents/sample-cases.csv @@ -0,0 +1,5 @@ +Case ID,Requirement,Scenario,Expected,Owner +TC-001,REQ-NC-017,Review left idle for 31 minutes,Case returns to the queue,QA +TC-002,REQ-NC-018,Screening call fails twice,Third attempt succeeds,QA +TC-003,REQ-NC-019,Empty name submitted,Rejected with NC-100,QA +TC-004,REQ-NC-017,Review completed in time,Decision is recorded,QA diff --git a/fixtures/documents/sample-design.html b/fixtures/documents/sample-design.html new file mode 100644 index 0000000..bb95d58 --- /dev/null +++ b/fixtures/documents/sample-design.html @@ -0,0 +1,39 @@ +<!doctype html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <title>NameCheck Design Note + + + + +
+

NameCheck Design Note

+

This note describes the internal design of the NameCheck screening service. + All of it is invented for testing.

+ +
+

Interfaces

+

Screening is submitted with POST /name-check and the outcome is read with + GET /name-check/{caseId}.

+
    +
  • Requests are validated before the watch list is consulted.
    • +
  • An empty name is rejected with NC-100.
    • +
+ 
curl -X POST https://localhost/name-check -d '{"name":"example"}'
+
+ +
+

Error codes

+ + + + +
CodeMeaning
NC-100The submitted name was empty
NC-101The watch list was unavailable
+
+ + +

Superseded paragraph kept out of view.

+
+ + diff --git a/fixtures/documents/sample-design.pdf b/fixtures/documents/sample-design.pdf new file mode 100644 index 0000000..2ca2eba Binary files /dev/null and b/fixtures/documents/sample-design.pdf differ diff --git a/fixtures/documents/sample-encrypted.pdf b/fixtures/documents/sample-encrypted.pdf new file mode 100644 index 0000000..a0df11d Binary files /dev/null and b/fixtures/documents/sample-encrypted.pdf differ diff --git a/fixtures/documents/sample-openapi.yaml b/fixtures/documents/sample-openapi.yaml new file mode 100644 index 0000000..cecf561 --- /dev/null +++ b/fixtures/documents/sample-openapi.yaml @@ -0,0 +1,64 @@ +openapi: 3.0.3 +info: + title: NameCheck API + version: 2.4.0 + description: Screening API for the invented NameCheck service. +servers: + - url: https://localhost/api +paths: + /name-check: + post: + operationId: submitScreening + summary: Submit a name for screening + description: Returns a case id that the caller polls for the decision. + tags: [screening] + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ScreeningRequest' + responses: + '202': + description: Screening accepted + content: + application/json: + schema: + $ref: '#/components/schemas/ScreeningCase' + '400': + description: The submitted name was empty (NC-100) + /name-check/{caseId}: + parameters: + - name: caseId + in: path + required: true + description: The case identifier returned by submitScreening. + schema: + type: string + get: + operationId: getScreeningResult + summary: Read a screening decision + responses: + '200': + description: The decision + '404': + description: No such case +components: + schemas: + ScreeningRequest: + type: object + required: [name] + properties: + name: + type: string + minLength: 1 + countryCode: + type: string + ScreeningCase: + type: object + properties: + caseId: + type: string + status: + type: string + enum: [pending, cleared, review] diff --git a/fixtures/documents/sample-requirements.docx b/fixtures/documents/sample-requirements.docx new file mode 100644 index 0000000..4ac0632 Binary files /dev/null and b/fixtures/documents/sample-requirements.docx differ diff --git a/fixtures/documents/sample-requirements.md b/fixtures/documents/sample-requirements.md new file mode 100644 index 0000000..f010a07 --- /dev/null +++ b/fixtures/documents/sample-requirements.md @@ -0,0 +1,47 @@ +# NameCheck Requirements + +The NameCheck service screens submitted names against a watch list. Everything in +this file is invented for testing; it describes no real system. + +## 2. Scope + +NameCheck runs as part of the onboarding flow. It is reached over HTTP and it +publishes an audit event for every decision. + +## 4. Functional Requirements + +### 4.3 Manual review + +REQ-NC-017: a manual review must time out after 30 minutes, and the case must +return to the shared review queue when it does. + +REQ-NC-018: the client retries a failed screening call three times before it +reports an error to the caller. + +- The screening endpoint is POST /name-check. +- The result endpoint is GET /name-check/{caseId}. +- The review timeout is configured by `namecheck.review.timeout.minutes`. +- Decisions are published to the `namecheck.decisions` topic. + +```java +public final class NameCheckService { + public ScreeningResult screen(ScreeningRequest request) { + return watchList.match(request.normalizedName()); + } +} +``` + +### 4.4 Error codes + +| Code | Meaning | Retryable | +| --- | --- | --- | +| NC-100 | The submitted name was empty | no | +| NC-101 | The watch list was unavailable | yes | +| NC-102 | The review timed out | yes | + +> Ticket ABC-1234 tracks the rollout of the NC-102 retry behaviour. + +## 5. Data + +The `screening_case` table stores one row per screening decision. See +`sample-schema.sql` for its definition. diff --git a/fixtures/documents/sample-scanned.pdf b/fixtures/documents/sample-scanned.pdf new file mode 100644 index 0000000..41ada62 Binary files /dev/null and b/fixtures/documents/sample-scanned.pdf differ diff --git a/fixtures/documents/sample-schema.json b/fixtures/documents/sample-schema.json new file mode 100644 index 0000000..e057c6e --- /dev/null +++ b/fixtures/documents/sample-schema.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.invalid/schemas/screening-case.json", + "title": "ScreeningCase", + "description": "One NameCheck screening case. Invented for testing.", + "type": "object", + "required": ["caseId", "status"], + "properties": { + "caseId": { + "type": "string", + "pattern": "^NC-[0-9]{6}$", + "description": "The case identifier." + }, + "status": { + "$ref": "#/$defs/CaseStatus" + }, + "submittedName": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "reviewer": { + "$ref": "#/$defs/Reviewer" + }, + "externalProfile": { + "$ref": "https://example.invalid/schemas/reviewer-profile.json" + } + }, + "$defs": { + "CaseStatus": { + "type": "string", + "enum": ["pending", "cleared", "review", "rejected"], + "description": "The lifecycle status of a screening case." + }, + "Reviewer": { + "type": "object", + "required": ["id"], + "properties": { + "id": { "type": "string" }, + "team": { "type": "string", "default": "operations" } + } + } + } +} diff --git a/fixtures/documents/sample-schema.sql b/fixtures/documents/sample-schema.sql new file mode 100644 index 0000000..8d5885d --- /dev/null +++ b/fixtures/documents/sample-schema.sql @@ -0,0 +1,33 @@ +-- NameCheck persistence model. Invented for testing; no real system uses it. + +CREATE TABLE screening_case ( + case_id VARCHAR(32) NOT NULL, + submitted_name VARCHAR(200) NOT NULL, + status VARCHAR(16) NOT NULL DEFAULT 'pending', + reviewer_id VARCHAR(32), + created_at TIMESTAMP NOT NULL, + CONSTRAINT pk_screening_case PRIMARY KEY (case_id), + CONSTRAINT ck_screening_status CHECK (status IN ('pending','cleared','review','rejected')) +); + +CREATE INDEX idx_screening_case_status ON screening_case (status, created_at); + +CREATE TABLE watch_list_entry ( + entry_id VARCHAR(32) NOT NULL PRIMARY KEY, + match_name VARCHAR(200) NOT NULL, + source VARCHAR(64) NOT NULL +); + +CREATE VIEW open_screening_case AS + SELECT case_id, submitted_name, created_at + FROM screening_case + WHERE status IN ('pending', 'review'); + +-- Vendor-specific syntax the general parser is not expected to structure. It +-- must be kept verbatim with its exact line range, not dropped or guessed at. +CREATE OR REPLACE PACKAGE BODY namecheck_admin IS + PROCEDURE purge_cases(p_before IN DATE) IS + BEGIN + DELETE FROM screening_case WHERE created_at < p_before; + END purge_cases; +END namecheck_admin; diff --git a/fixtures/documents/sample-tests.xlsx b/fixtures/documents/sample-tests.xlsx new file mode 100644 index 0000000..6ff2a02 Binary files /dev/null and b/fixtures/documents/sample-tests.xlsx differ diff --git a/openmind/cli.py b/openmind/cli.py index 3034433..62ed30d 100644 --- a/openmind/cli.py +++ b/openmind/cli.py @@ -533,6 +533,269 @@ def cmd_asset_add(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, return EXIT_OK, payload +# --------------------------------------------------------------------------- +# Documents (OpenMind v2 Phase 3) +# --------------------------------------------------------------------------- +def cmd_document_add(args: argparse.Namespace, + out: Output) -> Tuple[int, Dict[str, Any]]: + """Append a document from anywhere on this machine. + + The import DECISION is part of the result, not a hidden detail: an exact + duplicate, a filename collision needing the user's choice, and a real new + revision are three different outcomes and are reported as such. + """ + from .runtime import get_runtime + from .domain.types import ImportStatus + + if sum(bool(x) for x in (args.asset, args.logical_key, args.new_asset)) > 1: + raise InvalidRequest( + "--asset, --logical-key and --new-asset are mutually exclusive: " + "each names a DIFFERENT target for these bytes, so combining them " + "has no single correct meaning", + details={"asset": args.asset, "logical_key": args.logical_key, + "new_asset": args.new_asset}) + + runtime = get_runtime() + if args.wait and not args.dry_run: + out.say("Starting the job worker...") + try: + result = runtime.documents.add_document( + args.workspace, args.path, asset_id=args.asset or "", + logical_key=args.logical_key or "", new_asset=bool(args.new_asset), + version_label=args.version_label or "", wait=args.wait, + timeout=args.timeout, dry_run=bool(args.dry_run)) + except OperationTimeout as exc: + out.warn(str(exc)) + payload = {"ok": False, "error": exc.as_dict()} + payload.update(exc.details) + return EXIT_TIMEOUT, payload + + payload = _ok(result) + status = result.get("status", "") + + # The outcome is decided BEFORE emitting. `emit` serializes the payload at + # the moment it is called, so setting ok/error afterwards would print + # `"ok": true` on stdout for a run that then exits non-zero — the exact + # mismatch a script would trust and get wrong. + exit_code = EXIT_OK + if status == ImportStatus.POSSIBLE_REVISION: + # Nothing was written and the user must choose. + exit_code = EXIT_DOMAIN_FAILURE + payload["ok"] = False + payload["error"] = { + "code": "possible_revision", + "message": result.get("reason", "a different document already uses " + "that key"), + "details": {"possible_asset_id": (result.get("possible_asset") + or {}).get("id", ""), + "guidance": result.get("guidance", {})}, + } + elif status == ImportStatus.UNSUPPORTED: + exit_code = EXIT_DOMAIN_FAILURE + payload["ok"] = False + payload["error"] = {"code": "unsupported_document", + "message": result.get("reason", "unsupported")} + elif result.get("waited") and not result.get("completed"): + exit_code = EXIT_JOB_FAILURE + payload["ok"] = False + payload["error"] = result.get("error") or { + "code": "job_failed", + "message": f"document import did not complete " + f"(status: {result.get('status_job')})"} + + def human(p: Dict[str, Any]) -> None: + print(f"{p['status']} {p.get('logical_key', '')}") + if p.get("reason"): + print(f" {p['reason']}") + guidance = p.get("guidance") or {} + if guidance: + print(" resolve it with ONE of:") + print(f" --asset {p['possible_asset']['id']}" + f" (these bytes are a new revision of that document)") + print(f" --new-asset" + f" (this is a different document; it will " + f"become {guidance['suggested_new_key']})") + print(f" --logical-key (name it yourself)") + report = p.get("import_report") or {} + if report: + print(f" parser: {report.get('parser')} " + f"({report.get('parse_status')})") + print(f" asset: {report.get('asset_id')}") + print(f" revision: {report.get('revision_id')} " + f"(seq {report.get('revision_sequence')})") + print(f" segments: {report.get('segments_created')} " + f"({report.get('blocks_indexed')} indexed)") + if report.get("version_label"): + print(f" version: {report['version_label']} " + f"({report.get('version_label_source')})") + for warning in report.get("warnings") or []: + print(f" warning: {warning['code']}: {warning['message']}") + for entry in report.get("unsupported_content") or []: + print(f" not read: {entry['kind']} x{entry['count']}") + related = report.get("related_candidates") or {} + if related.get("count"): + print(f" candidates: {related['count']} related CANDIDATE(s) " + f"— observed mentions, not confirmed relations:") + for candidate in related.get("top", []): + target = (candidate["target"].get("logical_key") + or candidate["target"].get("symbol") or "") + print(f" {candidate['confidence']:6} " + f"{candidate['candidate_type']:24} {target}") + elif p.get("job_id"): + print(f" job: {p['job_id']}") + + if payload.get("error"): + out.warn(payload["error"]["message"]) + out.emit(payload, human) + return exit_code, payload + + +def cmd_document_list(args: argparse.Namespace, + out: Output) -> Tuple[int, Dict[str, Any]]: + """List the workspace's document assets (bounded).""" + from .runtime import get_runtime + + result = get_runtime().documents.list_documents( + args.workspace, status=args.status, parser=args.parser, + state=args.state, limit=args.limit, offset=args.offset) + payload = _ok(result) + + def human(p: Dict[str, Any]) -> None: + print(f"{p['count']} of {p['total']} document(s):") + for record in p["documents"]: + info = record.get("document") or {} + print(f" {record['id']} {info.get('status', ''):11} " + f"{info.get('parser_name', ''):12} {record['logical_key']}") + + out.emit(payload, human) + return EXIT_OK, payload + + +def cmd_document_show(args: argparse.Namespace, + out: Output) -> Tuple[int, Dict[str, Any]]: + """Show one document: asset, current revision and parse summary.""" + from .runtime import get_runtime + + document = get_runtime().documents.get_document(args.workspace, args.asset) + payload = _ok({"document": document}) + + def human(p: Dict[str, Any]) -> None: + d = p["document"] + parse = d["parse"] + print(f"{d['logical_key']} ({d['id']})") + print(f" title: {parse['title']}") + print(f" parser: {parse['parser_name']} {parse['parser_version']} " + f"-> {parse['status']}") + print(f" media: {parse['media_type']}") + print(f" state: {d['state']} (source: {d['source_kind']})") + current = d.get("current_revision") or {} + if current: + print(f" revision: {current['id']} (seq {current['sequence']}, " + f"{current['segment_count']} segment(s))") + if current.get("version_label"): + print(f" version: {current['version_label']}") + print(f" indexed: {d['index']['chunk_count']} chunk(s)") + for key, value in sorted((parse.get("coverage") or {}).items()): + print(f" coverage.{key}: {value}") + for warning in parse.get("warnings") or []: + print(f" warning: {warning['code']}: {warning['message']}") + for entry in parse.get("unsupported_content") or []: + print(f" not read: {entry['kind']} x{entry['count']}") + + out.emit(payload, human) + return EXIT_OK, payload + + +def cmd_document_outline(args: argparse.Namespace, + out: Output) -> Tuple[int, Dict[str, Any]]: + """Print a bounded STRUCTURAL outline of a document revision.""" + from .runtime import get_runtime + + result = get_runtime().documents.get_outline( + args.workspace, args.revision, limit=args.limit) + payload = _ok(result) + + def human(p: Dict[str, Any]) -> None: + print(f"{p['count']} of {p['total']} block(s) in revision " + f"{p['revision_id']} ({p['parse']['parser_name']}):") + for entry in p["outline"]: + depth = len(entry.get("heading_path") or []) + indent = " " * min(depth, 6) + print(f" {indent}{entry['block_type']:12} {entry['preview']}") + + out.emit(payload, human) + return EXIT_OK, payload + + +def cmd_document_search(args: argparse.Namespace, + out: Output) -> Tuple[int, Dict[str, Any]]: + """Search the workspace's documents.""" + from .runtime import get_runtime + + result = get_runtime().documents.search( + args.workspace, args.query, limit=args.limit, parser=args.parser, + block_type=args.block_type, logical_key=args.logical_key) + payload = _ok(result) + + def human(p: Dict[str, Any]) -> None: + print(f"{p['count']} hit(s) [{p['query_mode']}]") + for hit in p["hits"]: + print(f" {hit['logical_key']} {hit['block_type']} " + f"({', '.join(hit['retrieval_sources'])})") + print(f" evidence {hit['evidence_id']}: {hit['excerpt'][:160]}") + + out.emit(payload, human) + return EXIT_OK, payload + + +def cmd_document_related(args: argparse.Namespace, + out: Output) -> Tuple[int, Dict[str, Any]]: + """Deterministic candidate associations for one document.""" + from .runtime import get_runtime + + result = get_runtime().documents.find_related_candidates( + args.workspace, args.asset, limit=args.limit) + payload = _ok(result) + + def human(p: Dict[str, Any]) -> None: + print(f"{p['count']} CANDIDATE(s) — observed mentions, NOT confirmed " + f"relations:") + for candidate in p["candidates"]: + target = (candidate["target"].get("logical_key") + or candidate["target"].get("symbol") or "") + print(f" {candidate['confidence']:6} " + f"{candidate['candidate_type']:26} {candidate['mention'][:28]:30} " + f"-> {target}") + print(f" {candidate['reason']}") + + out.emit(payload, human) + return EXIT_OK, payload + + +def cmd_knowledge_search(args: argparse.Namespace, + out: Output) -> Tuple[int, Dict[str, Any]]: + """Search code and documents together, reported separately.""" + from .runtime import get_runtime + + result = get_runtime().documents.search_knowledge( + args.workspace, args.query, code_limit=args.limit, + document_limit=args.limit) + payload = _ok(result) + + def human(p: Dict[str, Any]) -> None: + print(f"code: {p['grounding']['codeCount']} hit(s)") + for chunk in p["code"]["hits"]: + print(f" {chunk.get('file_path', '')} {chunk.get('symbol', '')}") + print(f"documents: {p['grounding']['documentCount']} hit(s)") + for hit in p["documents"]["hits"]: + print(f" {hit['logical_key']} {hit['block_type']}: " + f"{hit['excerpt'][:120]}") + print(p["grounding"]["note"]) + + out.emit(payload, human) + return EXIT_OK, payload + + def cmd_export(args: argparse.Namespace, out: Output) -> Tuple[int, Dict[str, Any]]: """Generate the deterministic ``.openmind`` artifact directory. @@ -784,6 +1047,104 @@ def build_parser() -> argparse.ArgumentParser: asset.set_defaults(func=None, _parser=asset) + # -- documents (v2 Phase 3) -------------------------------------------- + document = sub.add_parser("document", parents=[common], + help="append and query enterprise documents") + document_sub = document.add_subparsers(dest="document_command", + metavar="") + + d_add = document_sub.add_parser( + "add", parents=[common], + help="append a document from anywhere on this machine") + d_add.add_argument("--workspace", required=True, help="workspace id") + d_add.add_argument("--path", required=True, help="the document file") + # Mutually exclusive in MEANING as well as in the parser: each names a + # different target for the same bytes. + d_target = d_add.add_mutually_exclusive_group() + d_target.add_argument("--asset", metavar="ASSET_ID", + help="treat the bytes as a new revision of this asset") + d_target.add_argument("--logical-key", dest="logical_key", metavar="KEY", + help="find or create the asset with this key") + d_target.add_argument("--new-asset", dest="new_asset", action="store_true", + help="create a distinct asset with a deterministic key") + d_add.add_argument("--version-label", dest="version_label", metavar="LABEL", + help="explicit version label for the new revision") + d_add.add_argument("--wait", action="store_true", + help="wait for the import job to finish") + d_add.add_argument("--timeout", type=float, default=3600.0, + metavar="SECONDS", help="bound for --wait (default: 3600)") + d_add.add_argument("--dry-run", dest="dry_run", action="store_true", + help="report the import plan and store nothing") + d_add.set_defaults(func=cmd_document_add) + + d_list = document_sub.add_parser("list", parents=[common], + help="list document assets (bounded)") + d_list.add_argument("--workspace", required=True, help="workspace id") + d_list.add_argument("--status", help="filter by parse status " + "(parsed/partial/needs-ocr/...)") + d_list.add_argument("--parser", help="filter by parser name") + d_list.add_argument("--state", default="active", + help="asset state (default: active)") + d_list.add_argument("--limit", type=int, default=100, metavar="N", + help="max documents to return (default: 100)") + d_list.add_argument("--offset", type=int, default=0, metavar="N", + help="skip the first N (default: 0)") + d_list.set_defaults(func=cmd_document_list) + + d_show = document_sub.add_parser( + "show", parents=[common], + help="show one document, its current revision and parse summary") + d_show.add_argument("--workspace", required=True, help="workspace id") + d_show.add_argument("--asset", required=True, help="document asset id") + d_show.set_defaults(func=cmd_document_show) + + d_outline = document_sub.add_parser( + "outline", parents=[common], + help="bounded structural outline of a document revision") + d_outline.add_argument("--workspace", required=True, help="workspace id") + d_outline.add_argument("--revision", required=True, help="revision id") + d_outline.add_argument("--limit", type=int, default=500, metavar="N", + help="max blocks to return (default: 500)") + d_outline.set_defaults(func=cmd_document_outline) + + d_search = document_sub.add_parser("search", parents=[common], + help="search the workspace's documents") + d_search.add_argument("--workspace", required=True, help="workspace id") + d_search.add_argument("--query", required=True, help="the search query") + d_search.add_argument("--limit", type=int, default=20, metavar="N", + help="max hits to return (default: 20)") + d_search.add_argument("--parser", help="filter by parser name") + d_search.add_argument("--block-type", dest="block_type", + help="filter by block type") + d_search.add_argument("--logical-key", dest="logical_key", + help="restrict to one document") + d_search.set_defaults(func=cmd_document_search) + + d_related = document_sub.add_parser( + "related", parents=[common], + help="deterministic candidate associations for one document") + d_related.add_argument("--workspace", required=True, help="workspace id") + d_related.add_argument("--asset", required=True, help="document asset id") + d_related.add_argument("--limit", type=int, default=30, metavar="N", + help="max candidates to return (default: 30)") + d_related.set_defaults(func=cmd_document_related) + + document.set_defaults(func=None, _parser=document) + + knowledge = sub.add_parser("knowledge", parents=[common], + help="query code and documents together") + knowledge_sub = knowledge.add_subparsers(dest="knowledge_command", + metavar="") + k_search = knowledge_sub.add_parser( + "search", parents=[common], + help="search code and documents, reported separately") + k_search.add_argument("--workspace", required=True, help="workspace id") + k_search.add_argument("--query", required=True, help="the search query") + k_search.add_argument("--limit", type=int, default=12, metavar="N", + help="max hits per side (default: 12)") + k_search.set_defaults(func=cmd_knowledge_search) + knowledge.set_defaults(func=None, _parser=knowledge) + serve = sub.add_parser("serve", parents=[common], help="run the FastAPI web application") serve.add_argument("--host", default="127.0.0.1", diff --git a/openmind/config.py b/openmind/config.py index d0ae3ef..b4c2b1c 100644 --- a/openmind/config.py +++ b/openmind/config.py @@ -183,6 +183,70 @@ def _first_existing(*candidates: str) -> str: OCR_MAX_UPLOAD_BYTES = 50_000_000 # /ocr rejects larger image uploads (413) instead of buffering them +# --------------------------------------------------------------------------- +# Document ingestion policy (OpenMind v2 Phase 3) +# --------------------------------------------------------------------------- +# Code discovery and DOCUMENT discovery are separate policies on purpose. A +# `.pdf` or `.docx` must never reach walker.read_text (it would decode binary +# noise and index it as source), so no document extension is added to +# INDEX_EXTENSIONS; the document walk has its own extension set, its own size +# limit, and its own parser plane. +CODE_INDEX_EXTENSIONS = INDEX_EXTENSIONS # readable alias; same set, unchanged + +TEXT_DOCUMENT_EXTENSIONS = { + ".md", ".markdown", ".rst", ".adoc", ".asciidoc", ".txt", +} +BINARY_DOCUMENT_EXTENSIONS = {".docx", ".pdf", ".xlsx"} +STRUCTURED_DOCUMENT_EXTENSIONS = {".csv", ".tsv"} + +#: Everything a document parser can claim when a user ATTACHES a file explicitly. +#: Wider than the discovery set: `.html`, `.yaml`, `.json` and `.sql` documents +#: are perfectly parseable on demand. +DOCUMENT_EXTENSIONS = (TEXT_DOCUMENT_EXTENSIONS | BINARY_DOCUMENT_EXTENSIONS + | STRUCTURED_DOCUMENT_EXTENSIONS + | {".html", ".htm", ".yaml", ".yml", ".json", ".sql"}) + +#: What a WORKSPACE walk discovers as a document. The subtraction is +#: load-bearing: a file the code pipeline already owns (.html/.yaml/.json/.sql) +#: must not also be ingested as a document, or one file would become two Assets. +#: Those formats stay reachable through `openmind document add`. +DOCUMENT_DISCOVERY_EXTENSIONS = ( + (TEXT_DOCUMENT_EXTENSIONS | BINARY_DOCUMENT_EXTENSIONS + | STRUCTURED_DOCUMENT_EXTENSIONS) - CODE_INDEX_EXTENSIONS) + + +def _int_env(name: str, default: int) -> int: + """An OPENMIND_* integer override, ignoring anything unparseable or <= 0 — + a typo must not silently disable a safety limit.""" + try: + value = int(os.environ.get(name, "").strip()) + except (TypeError, ValueError): + return default + return value if value > 0 else default + + +# Parser resource limits. Every one is enforced by the parser itself, and hitting +# one produces status='partial' + an exact warning naming the limit — never a +# silent truncation. Documents are untrusted input; these are the blast radius. +DOCUMENT_MAX_BYTES = _int_env("OPENMIND_DOCUMENT_MAX_BYTES", 25_000_000) +PDF_MAX_PAGES = _int_env("OPENMIND_PDF_MAX_PAGES", 2_000) +DOCX_MAX_PARAGRAPHS = _int_env("OPENMIND_DOCX_MAX_PARAGRAPHS", 50_000) +DOCX_MAX_TABLES = _int_env("OPENMIND_DOCX_MAX_TABLES", 2_000) +XLSX_MAX_SHEETS = _int_env("OPENMIND_XLSX_MAX_SHEETS", 100) +XLSX_MAX_ROWS_PER_SHEET = _int_env("OPENMIND_XLSX_MAX_ROWS_PER_SHEET", 20_000) +XLSX_MAX_CELLS = _int_env("OPENMIND_XLSX_MAX_CELLS", 500_000) +CSV_MAX_ROWS = _int_env("OPENMIND_CSV_MAX_ROWS", 50_000) +DOCUMENT_MAX_BLOCKS = _int_env("OPENMIND_DOCUMENT_MAX_BLOCKS", 20_000) +DOCUMENT_MAX_BLOCK_CHARS = _int_env("OPENMIND_DOCUMENT_MAX_BLOCK_CHARS", 20_000) + +# ZIP package safety for the OOXML formats (DOCX/XLSX). A document archive is +# never extracted to a filesystem path; these bound what may be read INTO MEMORY +# from an already-validated archive. +ZIP_MAX_MEMBERS = _int_env("OPENMIND_ZIP_MAX_MEMBERS", 2_000) +ZIP_MAX_TOTAL_BYTES = _int_env("OPENMIND_ZIP_MAX_TOTAL_BYTES", 400_000_000) +ZIP_MAX_MEMBER_BYTES = _int_env("OPENMIND_ZIP_MAX_MEMBER_BYTES", 100_000_000) +ZIP_MAX_RATIO = _int_env("OPENMIND_ZIP_MAX_RATIO", 200) + # Template profiles (declarative framework/architecture lenses; see openmind.templates). # Built-ins ship with the package; users drop their own .yaml/.json into the # data-dir folder — same schema, user files override built-ins by name. diff --git a/openmind/db.py b/openmind/db.py index b98d4ba..d52625d 100644 --- a/openmind/db.py +++ b/openmind/db.py @@ -636,10 +636,27 @@ def _segment_row(row: sqlite3.Row) -> Dict[str, Any]: "symbol": row["symbol"], "content_hash": row["content_hash"], "content_mode": row["content_mode"], + # v0004. Empty for code segments (they resolve through the revision + # blob's line range); the block-blob hash for document segments. + "content_blob_hash": _column(row, "content_blob_hash", ""), "metadata": json.loads(row["metadata_json"]), } +def _column(row: sqlite3.Row, name: str, default: Any = None) -> Any: + """Read a column that a NEWER migration added. + + A row read through an older cached statement, or in a test that builds a + partial row dict, may not carry it. Returning the default keeps a read + working instead of raising IndexError deep inside a mapper. + """ + try: + value = row[name] + except (IndexError, KeyError): + return default + return default if value is None else value + + def _evidence_row(row: sqlite3.Row) -> Dict[str, Any]: return { "id": row["id"], @@ -922,6 +939,304 @@ def clear_workspace_assets(workspace_id: str) -> None: _c().commit() +# --------------------------------------------------------------------------- +# Document projections (OpenMind v2 Phase 3) +# +# `document_parses` is a derived parse record over an immutable Revision, and +# its PRESENCE for an Asset's current Revision is the definition of "this Asset +# is a document" — a recorded fact, never an inference. `document_index` names +# the vector chunks currently live for an Asset, so a new current Revision can +# replace exactly its predecessor's chunks. Both are workspace-scoped at the SQL +# level through a JOIN back to assets.workspace_id, exactly like the Phase 2 +# reads above. +# --------------------------------------------------------------------------- +def _document_parse_row(row: sqlite3.Row) -> Dict[str, Any]: + return { + "revision_id": row["revision_id"], + "parser_name": row["parser_name"], + "parser_version": row["parser_version"], + "schema_version": row["schema_version"], + "status": row["status"], + "title": row["title"], + "media_type": row["media_type"], + "metadata": json.loads(row["metadata_json"]), + "warnings": json.loads(row["warnings_json"]), + "unsupported_content": json.loads(row["unsupported_json"]), + "coverage": json.loads(row["coverage_json"]), + "structure_hash": row["structure_hash"], + "created_at": row["created_at"], + } + + +def get_document_parse(workspace_id: str, + revision_id: str) -> Optional[Dict[str, Any]]: + with _lock: + row = _c().execute( + "SELECT d.* FROM document_parses d " + "JOIN asset_revisions r ON d.revision_id=r.id " + "JOIN assets a ON r.asset_id=a.id " + "WHERE d.revision_id=? AND a.workspace_id=?", + (revision_id, workspace_id), + ).fetchone() + return _document_parse_row(row) if row else None + + +def list_document_assets(workspace_id: str, status: Optional[str] = None, + parser: Optional[str] = None, + state: Optional[str] = "active", + limit: int = 100, + offset: int = 0) -> List[Dict[str, Any]]: + """Assets whose CURRENT revision has a parse record, newest-updated first. + + The join to ``document_parses`` on the asset's current revision is what + makes "is a document" a fact rather than a guess from the file extension. + """ + q = ("SELECT a.*, d.parser_name AS d_parser, d.status AS d_status, " + "d.title AS d_title, d.media_type AS d_media, " + "d.coverage_json AS d_coverage, d.created_at AS d_parsed_at " + "FROM assets a " + "JOIN document_parses d ON d.revision_id = a.current_revision_id " + "WHERE a.workspace_id=?") + args: List[Any] = [workspace_id] + if state: + q += " AND a.state=?" + args.append(state) + if status: + q += " AND d.status=?" + args.append(status) + if parser: + q += " AND d.parser_name=?" + args.append(parser) + q += " ORDER BY a.logical_key LIMIT ? OFFSET ?" + args.extend([max(0, int(limit)), max(0, int(offset))]) + with _lock: + rows = _c().execute(q, args).fetchall() + out: List[Dict[str, Any]] = [] + for r in rows: + record = _asset_row(r) + record["document"] = { + "parser_name": r["d_parser"], "status": r["d_status"], + "title": r["d_title"], "media_type": r["d_media"], + "coverage": json.loads(r["d_coverage"] or "{}"), + "parsed_at": r["d_parsed_at"], + } + out.append(record) + return out + + +def count_document_assets(workspace_id: str, status: Optional[str] = None, + parser: Optional[str] = None, + state: Optional[str] = "active") -> int: + q = ("SELECT COUNT(*) FROM assets a " + "JOIN document_parses d ON d.revision_id = a.current_revision_id " + "WHERE a.workspace_id=?") + args: List[Any] = [workspace_id] + if state: + q += " AND a.state=?" + args.append(state) + if status: + q += " AND d.status=?" + args.append(status) + if parser: + q += " AND d.parser_name=?" + args.append(parser) + with _lock: + row = _c().execute(q, args).fetchone() + return int(row[0]) if row else 0 + + +def find_document_revision_by_content_hash( + workspace_id: str, content_hash: str) -> Optional[Dict[str, Any]]: + """The first DOCUMENT revision in this workspace with exactly these bytes. + + This is the duplicate gate: identical content already ingested as a document + must create no job, no revision and no vector duplicate. Restricted to + revisions that HAVE a parse record, so a code file that happens to share a + hash never masquerades as an already-imported document. + """ + if not content_hash: + return None + with _lock: + row = _c().execute( + "SELECT r.*, a.id AS a_id, a.logical_key AS a_key, " + "a.state AS a_state FROM asset_revisions r " + "JOIN assets a ON r.asset_id=a.id " + "JOIN document_parses d ON d.revision_id=r.id " + "WHERE a.workspace_id=? AND r.content_hash=? " + "ORDER BY r.created_at LIMIT 1", + (workspace_id, content_hash), + ).fetchone() + if not row: + return None + out = _revision_row(row) + out["asset_id"] = row["a_id"] + out["logical_key"] = row["a_key"] + out["asset_state"] = row["a_state"] + return out + + +def is_document_asset(workspace_id: str, asset_id: str) -> bool: + """Whether this Asset's CURRENT revision carries a parse record.""" + with _lock: + row = _c().execute( + "SELECT 1 FROM assets a " + "JOIN document_parses d ON d.revision_id = a.current_revision_id " + "WHERE a.id=? AND a.workspace_id=?", (asset_id, workspace_id), + ).fetchone() + return row is not None + + +def list_workspace_symbols(workspace_id: str, exclude_asset: str = "", + limit: int = 20_000) -> Dict[str, Dict[str, Any]]: + """symbol -> {segment_id, evidence_id, asset_id, logical_key, asset_type} + across the workspace's CURRENT revisions, in one query. + + Only current revisions: a candidate must point at what the workspace knows + NOW, not at a symbol that existed three revisions ago. Bounded, and the first + occurrence of a symbol wins so the result is stable across calls. + """ + with _lock: + rows = _c().execute( + "SELECT s.symbol AS sym, s.id AS sid, s.segment_type AS stype, " + "a.id AS aid, a.logical_key AS lk, a.asset_type AS atype, " + "e.id AS eid FROM segments s " + "JOIN assets a ON a.current_revision_id = s.revision_id " + "LEFT JOIN evidence e ON e.segment_id = s.id " + "WHERE a.workspace_id=? AND a.state='active' AND s.symbol != '' " + "ORDER BY a.logical_key, s.ordinal LIMIT ?", + (workspace_id, max(0, int(limit))), + ).fetchall() + #: Segment kinds whose symbol is a real CODE symbol. A `file` segment's + #: symbol is just the basename, and a document block's is its heading — both + #: are useful to look up verbatim but must never be split into aliases. + code_kinds = {"type", "method", "constructor"} + out: Dict[str, Dict[str, Any]] = {} + for r in rows: + if exclude_asset and r["aid"] == exclude_asset: + continue + symbol = r["sym"] + record = {"segment_id": r["sid"], "evidence_id": r["eid"] or "", + "asset_id": r["aid"], "logical_key": r["lk"], + "asset_type": r["atype"], "segment_type": r["stype"]} + if symbol not in out: + out[symbol] = record + if r["stype"] not in code_kinds: + continue + # A Java member segment's symbol is `pkg.Class#signature`; the bare class + # name is what a document actually writes, so it is aliased too. ONLY for + # code segments: aliasing a `file` segment would turn `screening.sql` + # into the "symbol" `sql`, which matches every mention of the word. + base = symbol.split("#", 1)[0].rsplit(".", 1)[-1] + if base and base != symbol and base not in out: + out[base] = dict(record) + return out + + +def get_document_index(workspace_id: str, + asset_id: str) -> Optional[Dict[str, Any]]: + with _lock: + row = _c().execute( + "SELECT * FROM document_index WHERE workspace_id=? AND asset_id=?", + (workspace_id, asset_id), + ).fetchone() + if not row: + return None + return {"workspace_id": row["workspace_id"], "asset_id": row["asset_id"], + "revision_id": row["revision_id"], + "chunk_ids": json.loads(row["chunk_ids_json"]), + "updated_at": row["updated_at"]} + + +def list_document_index(workspace_id: str) -> Dict[str, Dict[str, Any]]: + """asset_id -> index record for the whole workspace, in one query.""" + with _lock: + rows = _c().execute( + "SELECT * FROM document_index WHERE workspace_id=?", + (workspace_id,)).fetchall() + return {r["asset_id"]: {"asset_id": r["asset_id"], + "revision_id": r["revision_id"], + "chunk_ids": json.loads(r["chunk_ids_json"]), + "updated_at": r["updated_at"]} for r in rows} + + +def upsert_document_index(workspace_id: str, asset_id: str, revision_id: str, + chunk_ids: List[str]) -> None: + """Record an Asset's live chunk list OUTSIDE a revision commit. + + The normal path writes this inside ``commit_revision``'s transaction. This + exists for the one case with no new revision to commit: a document that was + removed and has reappeared unchanged, whose projection has to be rebuilt + against its EXISTING current revision. + """ + with _lock: + _c().execute( + "INSERT INTO document_index (workspace_id,asset_id,revision_id," + "chunk_ids_json,updated_at) VALUES (?,?,?,?,?) " + "ON CONFLICT(workspace_id,asset_id) DO UPDATE SET " + "revision_id=excluded.revision_id, " + "chunk_ids_json=excluded.chunk_ids_json, " + "updated_at=excluded.updated_at", + (workspace_id, asset_id, revision_id, json.dumps(list(chunk_ids)), + now())) + _c().commit() + + +def delete_document_index(workspace_id: str, asset_id: str) -> None: + """Forget an Asset's live chunk list (its document was removed). + + The Asset, its Revisions, Segments, Evidence and blobs are all PRESERVED — + only the active retrieval projection goes. + """ + with _lock: + _c().execute( + "DELETE FROM document_index WHERE workspace_id=? AND asset_id=?", + (workspace_id, asset_id)) + _c().commit() + + +def document_stats(workspace_id: str) -> Dict[str, Any]: + """Aggregate document counts for the workspace, by parse status.""" + with _lock: + c = _c() + total = c.execute( + "SELECT COUNT(*) FROM assets a JOIN document_parses d " + "ON d.revision_id = a.current_revision_id WHERE a.workspace_id=?", + (workspace_id,)).fetchone()[0] + rows = c.execute( + "SELECT d.status AS s, COUNT(*) AS n FROM assets a " + "JOIN document_parses d ON d.revision_id = a.current_revision_id " + "WHERE a.workspace_id=? GROUP BY d.status", (workspace_id,)).fetchall() + indexed = c.execute( + "SELECT COUNT(*) FROM document_index WHERE workspace_id=?", + (workspace_id,)).fetchone()[0] + return {"documents_total": int(total), "documents_indexed": int(indexed), + "by_status": {r["s"]: int(r["n"]) for r in rows}} + + +# --------------------------------------------------------------------------- +# Job payload (v0004) — a job's structured input, kept out of the free `path` +# column (which already means something different per job type) and guaranteed +# to carry NO absolute machine path. +# --------------------------------------------------------------------------- +def get_job_payload(job_id: str) -> Dict[str, Any]: + with _lock: + row = _c().execute("SELECT payload_json FROM jobs WHERE job_id=?", + (job_id,)).fetchone() + if not row: + return {} + try: + return json.loads(row["payload_json"] or "{}") + except Exception: + return {} + + +def set_job_payload(job_id: str, payload: Dict[str, Any]) -> None: + with _lock: + _c().execute("UPDATE jobs SET payload_json=?, updated_at=? " + "WHERE job_id=?", (json.dumps(payload), now(), job_id)) + _c().commit() + + def commit_revision( workspace_id: str, logical_key: str, @@ -941,6 +1256,10 @@ def commit_revision( revision_metadata: Optional[Dict[str, Any]] = None, asset_metadata: Optional[Dict[str, Any]] = None, asset_state: str = "active", + document_parse: Optional[Dict[str, Any]] = None, + document_chunk_ids: Optional[List[str]] = None, + asset_id: Optional[str] = None, + revision_id: Optional[str] = None, ) -> Dict[str, Any]: """The single transactional writer for the Asset model. @@ -959,6 +1278,25 @@ def commit_revision( ``{locator, excerpt, content_hash}``. Returns a summary: ``{asset_id, revision, revision_created, asset_created, reactivated, segments_created, evidence_created}``. + + DOCUMENT PROJECTIONS (v2 Phase 3) + -------------------------------- + ``document_parse`` and ``document_chunk_ids`` are written INSIDE the same + transaction, deliberately. A ``document_parses`` row committed separately + could survive a rolled-back revision (a parse record for a revision that + does not exist), and a ``document_index`` row committed separately could + point at chunks belonging to a revision that never became current. Both are + keyed to the revision this call mints, so either everything lands or nothing + does. + + PRE-MINTED IDS + -------------- + ``asset_id`` / ``revision_id`` (and a per-draft ``id`` on a segment or its + evidence) let a caller supply the identifiers instead of having them minted + here. The document pipeline needs that: its vector chunks carry the segment + and evidence ids in their metadata and are upserted BEFORE the transaction, + so those ids have to exist first. A supplied ``asset_id`` is used only when + the Asset does not already exist. """ ts = now() rev_meta = json.dumps(revision_metadata or {}) @@ -974,7 +1312,7 @@ def commit_revision( asset_created = False if existing is not None: - asset_id = existing["id"] + resolved_asset_id = existing["id"] cur_rev_id = existing["current_revision_id"] cur_hash = None if cur_rev_id: @@ -989,7 +1327,7 @@ def commit_revision( reactivated = existing["state"] == "removed" c.execute( "UPDATE assets SET state=?, updated_at=? WHERE id=?", - (asset_state, ts, asset_id)) + (asset_state, ts, resolved_asset_id)) c.commit() else: c.commit() @@ -997,7 +1335,7 @@ def commit_revision( "SELECT * FROM asset_revisions WHERE id=?", (cur_rev_id,)).fetchone() return { - "asset_id": asset_id, + "asset_id": resolved_asset_id, "revision": _revision_row(rev) if rev else None, "revision_created": False, "asset_created": False, "reactivated": reactivated, @@ -1008,7 +1346,7 @@ def commit_revision( reactivated = existing["state"] == "removed" supersedes = cur_rev_id else: - asset_id = new_id("a_") + resolved_asset_id = asset_id or new_id("a_") asset_created = True supersedes = None c.execute( @@ -1016,17 +1354,19 @@ def commit_revision( "title,source_kind,source_path,media_type,state," "current_revision_id,metadata_json,created_at,updated_at) " "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", - (asset_id, workspace_id, logical_key, asset_type, title, - source_kind, source_path, media_type, asset_state, None, - json.dumps(asset_metadata or {}), ts, ts)) + (resolved_asset_id, workspace_id, logical_key, asset_type, + title, source_kind, source_path, media_type, asset_state, + None, json.dumps(asset_metadata or {}), ts, ts)) # next dense sequence for this asset seq_row = c.execute( - "SELECT COALESCE(MAX(sequence),0) FROM asset_revisions WHERE asset_id=?", - (asset_id,)).fetchone() + "SELECT COALESCE(MAX(sequence),0) FROM asset_revisions " + "WHERE asset_id=?", (resolved_asset_id,)).fetchone() sequence = int(seq_row[0]) + 1 - revision_id = new_id("r_") + new_revision_id = revision_id or new_id("r_") + revision_id = new_revision_id + asset_id = resolved_asset_id c.execute( "INSERT INTO asset_revisions (id,asset_id,sequence,content_hash," "content_size,content_blob_hash,status,version_label,source_commit," @@ -1039,15 +1379,16 @@ def commit_revision( seg_count = 0 ev_count = 0 for seg in segments: - seg_id = new_id("s_") + seg_id = seg.get("id") or new_id("s_") c.execute( "INSERT INTO segments (id,revision_id,segment_key,segment_type," "ordinal,start_line,end_line,symbol,content_hash,content_mode," - "metadata_json) VALUES (?,?,?,?,?,?,?,?,?,?,?)", + "content_blob_hash,metadata_json) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", (seg_id, revision_id, seg["segment_key"], seg["segment_type"], int(seg["ordinal"]), seg.get("start_line"), seg.get("end_line"), seg.get("symbol", ""), seg.get("content_hash", ""), seg.get("content_mode", "verbatim"), + seg.get("content_blob_hash", ""), json.dumps(seg.get("metadata") or {}))) seg_count += 1 ev = seg.get("evidence") @@ -1055,11 +1396,41 @@ def commit_revision( c.execute( "INSERT INTO evidence (id,revision_id,segment_id,locator_json," "excerpt,content_hash,created_at) VALUES (?,?,?,?,?,?,?)", - (new_id("e_"), revision_id, seg_id, + (ev.get("id") or new_id("e_"), revision_id, seg_id, json.dumps(ev.get("locator") or {}), ev.get("excerpt", ""), ev.get("content_hash", ""), ts)) ev_count += 1 + if document_parse is not None: + c.execute( + "INSERT INTO document_parses (revision_id,parser_name," + "parser_version,schema_version,status,title,media_type," + "metadata_json,warnings_json,unsupported_json,coverage_json," + "structure_hash,created_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (revision_id, + document_parse.get("parser_name", ""), + document_parse.get("parser_version", ""), + document_parse.get("schema_version", ""), + document_parse.get("status", ""), + document_parse.get("title", ""), + document_parse.get("media_type", ""), + json.dumps(document_parse.get("metadata") or {}), + json.dumps(document_parse.get("warnings") or []), + json.dumps(document_parse.get("unsupported_content") or []), + json.dumps(document_parse.get("coverage") or {}), + document_parse.get("structure_hash", ""), ts)) + + if document_chunk_ids is not None: + c.execute( + "INSERT INTO document_index (workspace_id,asset_id," + "revision_id,chunk_ids_json,updated_at) VALUES (?,?,?,?,?) " + "ON CONFLICT(workspace_id,asset_id) DO UPDATE SET " + "revision_id=excluded.revision_id, " + "chunk_ids_json=excluded.chunk_ids_json, " + "updated_at=excluded.updated_at", + (workspace_id, asset_id, revision_id, + json.dumps(list(document_chunk_ids)), ts)) + if supersedes: c.execute( "UPDATE asset_revisions SET status='superseded' WHERE id=?", diff --git a/openmind/document_rag.py b/openmind/document_rag.py new file mode 100644 index 0000000..fcdb4a8 --- /dev/null +++ b/openmind/document_rag.py @@ -0,0 +1,414 @@ +"""Document retrieval over the per-workspace ``documents_`` collection. + +WHY A SEPARATE MODULE AND A SEPARATE COLLECTION +----------------------------------------------- +:mod:`openmind.rag` and the MCP ``search`` tool are a stable external contract: +their chunk ids, metadata keys and response shape are what editor clients depend +on. Adding document chunks to the code collection would silently change what +every existing ``search`` call returns. So documents get their own collection and +their own retrieval, and ``search`` stays code-oriented and byte-for-byte +unchanged. + +THE RANKING RULE THAT MATTERS +----------------------------- +An exact **Requirement ID, API path, error code or configuration key must never +be displaced by a merely embedding-similar result.** Someone searching +``REQ-NC-017`` wants that requirement, not a paragraph that reads like it. So +retrieval runs three legs and fuses them: + +* **vector** — conceptual similarity (candidates only); +* **lexical** — token-BOUNDARY matching via :mod:`openmind.tokenmatch`, so + ``REQ-NC-017`` never matches ``REQ-NC-0170`` and ``ack`` never matches + ``acked``; +* **exact identifier** — when the whole query IS an identifier, a hit that + contains it as a complete token is PROMOTED above every semantic-only hit, + rather than merely scored higher. + +Fusion is Reciprocal Rank Fusion, the same deterministic scheme the code RAG +uses, so the two behave alike where they overlap. + +EVERY HIT IS CITABLE +-------------------- +A hit carries its ``segment_id`` and ``evidence_id``, so a caller can go straight +from a search result to the exact stored block text via +``AssetService.get_evidence`` — no re-parsing, no guessing which part of the +document the answer came from. +""" +from __future__ import annotations + +import re +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple + +from . import db, embeddings, tokenmatch, vectorstore + +#: Hard ceiling on returned hits, whatever a caller asks for. +MAX_HITS = 50 +#: How much of a block's text is returned as the excerpt. The full text is +#: available through get_evidence; a search response must stay bounded. +EXCERPT_CHARS = 400 + +#: Identifier shapes worth treating as exact even inside a longer sentence. +#: Deliberately narrow and explicit — each is a real convention, not a guess: +#: REQ-NC-017 / ABC-1234 requirement and ticket ids +#: NC-100 error codes +#: a.b.c dotted configuration keys and topic names +#: /name-check API paths +_ID_PATTERNS: Tuple[re.Pattern, ...] = ( + re.compile(r"\b[A-Z][A-Z0-9]{1,9}(?:-[A-Z0-9]{1,9}){1,3}\b"), + re.compile(r"\b[a-z][a-z0-9]*(?:\.[a-z0-9][a-z0-9_]*){2,}\b"), + re.compile(r"(? List[str]: + """Explicit identifiers inside a free-text query, in a stable order. + + ``"what is the manual review timeout for REQ-NC-017?"`` -> ``["REQ-NC-017"]``. + These are what get PROMOTED; ordinary words only inform the conceptual leg. + """ + found: List[str] = [] + seen: Set[str] = set() + for pattern in _ID_PATTERNS: + for match in pattern.findall(query or ""): + token = match.strip().rstrip(".,;:") + if token and token not in seen: + seen.add(token) + found.append(token) + return found[:10] + + +def extract_terms(query: str) -> List[str]: + """Lexical terms for the token-matching leg: the identifiers plus any + distinctive words. + + Two exclusions, both load-bearing: + + * stop-words, so a natural-language question does not lexically match every + document in the workspace; + * any word that is a COMPONENT of an identifier already extracted. Without + it, ``REQ-NC-017`` also contributes the bare term ``REQ`` — which + token-matches ``REQ-NC-018`` and ``REQ-NC-019`` — and the precise query + quietly turns into "every requirement". The same applies to + ``namecheck.review.timeout.minutes``, whose four components would match + most of the corpus. + """ + terms = extract_identifiers(query) + seen = set(terms) + components = {part.lower() + for identifier in terms + for part in re.split(r"[^A-Za-z0-9]+", identifier) if part} + for word in _WORD_RE.findall(query or ""): + if word.lower() in _STOP or word in seen or word.lower() in components: + continue + seen.add(word) + terms.append(word) + return terms[:12] + + +def matches_term(text: str, term: str, *, case_sensitive: bool = True) -> bool: + """Whether *term* occurs in *text* as a COMPLETE token. + + Delegates to :mod:`openmind.tokenmatch` for identifiers and dotted literals. + API paths need their own rule: ``/`` is not an identifier character, so + ``tokenmatch`` cannot see ``/name-check`` as one token, and a plain substring + test would match ``/name-checker`` too. The boundary here is "the next + character is not part of a path segment", which keeps ``/name-check`` from + matching ``/name-check-v2`` while still matching ``/name-check/{caseId}`` — + the sub-path genuinely IS under the queried path. + """ + if not term: + return False + if not term.startswith("/"): + return tokenmatch.match_kind(text, term, + case_sensitive=case_sensitive) is not None + haystack = text if case_sensitive else text.lower() + needle = term if case_sensitive else term.lower() + start = 0 + while True: + at = haystack.find(needle, start) + if at < 0: + return False + before = haystack[at - 1] if at > 0 else " " + after_index = at + len(needle) + after = haystack[after_index] if after_index < len(haystack) else " " + if not (before.isalnum() or before in "-_") and \ + not (after.isalnum() or after in "-_"): + return True + start = at + 1 + + +def _rrf(rank_lists: Iterable[List[str]], k0: int = 60) -> Dict[str, float]: + scores: Dict[str, float] = {} + for lst in rank_lists: + for rank, cid in enumerate(lst): + scores[cid] = scores.get(cid, 0.0) + 1.0 / (k0 + rank + 1) + return scores + + +def _where(asset_type: Optional[str], parser: Optional[str], + block_type: Optional[str], logical_key: Optional[str], + asset_id: Optional[str]) -> Optional[Dict[str, Any]]: + """A Chroma ``where`` clause for the supported filters, or None. + + Chroma needs ``$and`` for more than one condition; a single condition must be + passed bare, which is why this is not just a dict comprehension. + """ + clauses = [] + for field, value in (("asset_type", asset_type), ("parser_name", parser), + ("block_type", block_type), + ("logical_key", logical_key), ("asset_id", asset_id)): + if value: + clauses.append({field: {"$eq": value}}) + if not clauses: + return None + return clauses[0] if len(clauses) == 1 else {"$and": clauses} + + +def _active_asset_ids(workspace_id: str) -> Set[str]: + """Assets whose document projection is live AND whose Asset is active. + + A removed document keeps its Revisions, Segments, Evidence and blobs, but its + chunks are deleted from the collection — this set is the backstop for the + window between those two facts, and for any chunk a failed commit stranded. + """ + index = db.list_document_index(workspace_id) + if not index: + return set() + active: Set[str] = set() + for asset_id in index: + asset = db.get_asset(workspace_id, asset_id) + if asset and asset.get("state") == "active": + active.add(asset_id) + return active + + +def _excerpt(document: str) -> str: + """Strip the structural header the chunk was embedded with, and bound it. + + The header (``// document: ...``) is there to put the title and section into + the EMBEDDING; showing it back to a reader as if it were document content + would be misleading. + """ + body = document or "" + lines = body.split("\n") + start = 0 + for i, line in enumerate(lines): + if line.startswith("// "): + start = i + 1 + continue + break + text = "\n".join(lines[start:]).strip() + return text[:EXCERPT_CHARS] + + +def search(workspace_id: str, query: str, limit: int = 20, *, + asset_type: Optional[str] = None, parser: Optional[str] = None, + block_type: Optional[str] = None, logical_key: Optional[str] = None, + asset_id: Optional[str] = None, + include_removed: bool = False, + case_sensitive: bool = True) -> Dict[str, Any]: + """Bounded, evidence-cited document retrieval. + + Returns ``{query, hits, count, identifiers, query_mode, filters, + grounding}``. Removed Assets are excluded unless *include_removed*. + """ + limit = max(1, min(int(limit or 20), MAX_HITS)) + store = vectorstore.get_documents_store(workspace_id) + where = _where(asset_type, parser, block_type, logical_key, asset_id) + allowed = None if include_removed else _active_asset_ids(workspace_id) + + pool: Dict[str, Dict[str, Any]] = {} + + def remember(cid: str, document: str, meta: Dict[str, Any], + source: str) -> Optional[Dict[str, Any]]: + if allowed is not None and meta.get("asset_id") not in allowed: + return None + record = pool.get(cid) + if record is None: + record = pool[cid] = {"doc": document, "meta": meta, + "sources": set(), "kinds": set()} + record["sources"].add(source) + return record + + # ---- vector leg (candidates only) ------------------------------------ + vector_rank: List[str] = [] + try: + qvec = embeddings.embed([query])[0].tolist() + result = store.query(qvec, n_results=max(limit * 3, 15), where=where) + for cid, doc, meta in zip(result["ids"], result["documents"], + result["metadatas"]): + if remember(cid, doc, meta, "vector") is not None: + vector_rank.append(cid) + except Exception: + # Retrieval must degrade, not fail: with no embedding backend the lexical + # and exact legs still answer, which is exactly the case an exact + # Requirement-ID lookup depends on. + vector_rank = [] + + # ---- lexical leg: TOKEN-BOUNDARY matching ---------------------------- + identifiers = extract_identifiers(query) + terms = extract_terms(query) + lexical_hits: Dict[str, int] = {} + exact_hits: Set[str] = set() + if terms: + scan = store.get(where=where) + for cid, doc, meta in zip(scan["ids"], scan["documents"], + scan["metadatas"]): + matched = 0 + for term in terms: + if matches_term(doc, term, case_sensitive=case_sensitive): + matched += 1 + if term in identifiers: + exact_hits.add(cid) + if matched: + record = remember(cid, doc, meta, "lexical") + if record is not None: + record["kinds"].add("token") + lexical_hits[cid] = matched + else: + exact_hits.discard(cid) + + lexical_rank = [cid for cid, _ in + sorted(lexical_hits.items(), key=lambda kv: (-kv[1], kv[0]))] + fused = _rrf([vector_rank, lexical_rank]) + + # When the WHOLE query is one identifier, only chunks that actually contain + # it are returned — an embedding-similar paragraph is not a hit for + # "REQ-NC-017", it is a different requirement. This mirrors the code RAG's + # exact-token mode, so both surfaces answer an identifier the same way. + bare_identifier = tokenmatch.is_exact_token_query(query) or ( + len(identifiers) == 1 and identifiers[0] == query.strip()) + if bare_identifier and lexical_hits: + candidates = [cid for cid in pool if cid in lexical_hits] + query_mode = "exact_token" + else: + candidates = list(pool.keys()) + query_mode = "exact_identifier" if exact_hits else "conceptual" + + # An exact identifier match is PROMOTED, not merely up-weighted: a semantic + # near-miss must never outrank the requirement the user literally named. + ordered = sorted( + candidates, + key=lambda cid: (0 if cid in exact_hits else 1, + -fused.get(cid, 0.0), + -lexical_hits.get(cid, 0), + cid)) + + hits = [_hit(cid, pool[cid], fused.get(cid, 0.0), cid in exact_hits) + for cid in ordered[:limit]] + return { + "workspace_id": workspace_id, + "query": query, + "hits": hits, + "count": len(hits), + "identifiers": identifiers, + "query_mode": query_mode, + "filters": {"asset_type": asset_type, "parser": parser, + "block_type": block_type, "logical_key": logical_key, + "asset_id": asset_id, "include_removed": include_removed}, + "grounding": tokenmatch.GROUNDING_NOTE, + } + + +def _hit(chunk_id: str, record: Dict[str, Any], score: float, + exact: bool) -> Dict[str, Any]: + meta = record["meta"] or {} + sources = sorted(record["sources"]) + if exact: + sources = sorted(set(sources) | {"exact-identifier"}) + heading = str(meta.get("heading_path") or "") + return { + "chunk_id": chunk_id, + "asset_id": meta.get("asset_id", ""), + "revision_id": meta.get("revision_id", ""), + "segment_id": meta.get("segment_id", ""), + "evidence_id": meta.get("evidence_id", ""), + "logical_key": meta.get("logical_key", ""), + "title": meta.get("title", ""), + "asset_type": meta.get("asset_type", ""), + "block_type": meta.get("block_type", ""), + "parser": meta.get("parser_name", ""), + "heading_path": [p for p in heading.split(" > ") if p], + "locator": _locator_from_meta(meta), + "excerpt": _excerpt(record["doc"]), + "score": round(float(score), 6), + "retrieval_sources": sources, + } + + +def _locator_from_meta(meta: Dict[str, Any]) -> Dict[str, Any]: + """Rebuild the block's portable locator from the chunk metadata. + + Chroma stores scalars only, so the locator is flattened on write and + reassembled here — with only the fields that are meaningful for its kind, so + a spreadsheet hit does not come back carrying ``page: 0``. + """ + kind = str(meta.get("locator_kind") or "") + locator: Dict[str, Any] = {"kind": kind, + "document": meta.get("logical_key", "")} + if kind == "text-range": + locator["startLine"] = int(meta.get("start_line") or 0) + locator["endLine"] = int(meta.get("end_line") or 0) + elif kind == "pdf-block": + locator["page"] = int(meta.get("page") or 0) + elif kind == "spreadsheet-range": + locator["sheet"] = str(meta.get("sheet") or "") + elif kind == "json-pointer": + locator["pointer"] = str(meta.get("json_pointer") or "") + heading = str(meta.get("heading_path") or "") + if heading: + locator["headingPath"] = [p for p in heading.split(" > ") if p] + return locator + + +def search_knowledge(workspace_ids: List[str], query: str, *, + code_limit: int = 12, + document_limit: int = 12) -> Dict[str, Any]: + """Code and document candidates, returned SEPARATELY. + + The separation is the contract. A document hit and a code hit appearing + together is retrieval, not a relationship: this never claims that the + document *implements*, *refines* or *verifies* the code. Establishing that + needs semantic verification, which is Phase 4. + """ + from . import rag + + pids = [p for p in workspace_ids if p] + code: Dict[str, Any] = {"hits": []} + try: + result = rag.retrieve(pids, query, k=code_limit) + code = {"hits": result.get("code_chunks", [])[:code_limit], + "query_mode": result.get("query_mode", "")} + except Exception as exc: + code = {"hits": [], "error": f"{type(exc).__name__}: {exc}"} + + documents: Dict[str, Any] = {"hits": []} + doc_hits: List[Dict[str, Any]] = [] + for pid in pids: + found = search(pid, query, limit=document_limit) + doc_hits.extend(found["hits"]) + doc_hits.sort(key=lambda h: -h["score"]) + documents = {"hits": doc_hits[:document_limit]} + + return { + "query": query, + "workspace_ids": pids, + "code": code, + "documents": documents, + "grounding": { + "codeCount": len(code["hits"]), + "documentCount": len(documents["hits"]), + "note": ("Code and document results are retrieved independently. " + "A document result appearing beside a code result is NOT a " + "claim that one implements, refines or verifies the other."), + }, + } + + +__all__ = ["MAX_HITS", "EXCERPT_CHARS", "extract_identifiers", "extract_terms", + "search", "search_knowledge"] diff --git a/openmind/documents/__init__.py b/openmind/documents/__init__.py new file mode 100644 index 0000000..e527f66 --- /dev/null +++ b/openmind/documents/__init__.py @@ -0,0 +1,60 @@ +"""Deterministic enterprise-document ingestion (OpenMind v2 Phase 3). + +Importing this package imports NO parser dependency: the registry stores module +paths and imports a parser only when a probe matches it, and each parser imports +its third-party library inside ``parse()``. That is what keeps the +dependency-free ``.openmind`` artifact export runnable on a machine with none of +the document packages installed. + +No model is involved anywhere in here. Parsing is byte-deterministic, structure +is read from the file's own markup, and nothing is inferred about meaning — +Requirement, Business Rule and Design Decision extraction are Phase 4. + + from openmind.documents import parse_bytes + result = parse_bytes(data, logical_key="documents/spec.md", + filename="spec.md") + result.status, result.title, len(result.blocks) +""" +from __future__ import annotations + +from typing import Any, Dict, Optional + +from .models import (DOCUMENT_SCHEMA_VERSION, DocumentBlock, DocumentLimits, + DocumentMetadata, DocumentParseContext, + DocumentParseWarning, DocumentProbe, ParsedDocument, + UnsupportedContent) +from .probe import probe_bytes +from .registry import (AmbiguousParser, DocumentParser, ParserEntry, + ParserRegistryError, list_parsers, parse, parser_names, + select) + + +def parse_bytes(content: bytes, *, logical_key: str, filename: str = "", + workspace_id: str = "", declared_media_type: str = "", + limits: Optional[DocumentLimits] = None, + parser_options: Optional[Dict[str, Any]] = None + ) -> ParsedDocument: + """Probe, select a parser and parse — the one call most callers need. + + Never raises for a bad document: every outcome is a :class:`ParsedDocument` + with an explicit status (``parsed``, ``partial``, ``needs-ocr``, + ``encrypted``, ``unsupported`` or ``failed``). + """ + context = DocumentParseContext( + logical_key=logical_key, filename=filename or logical_key, + workspace_id=workspace_id, declared_media_type=declared_media_type, + limits=limits or DocumentLimits.from_config(), + parser_options=dict(parser_options or {})) + probe = probe_bytes(content, filename=context.filename, + declared_media_type=declared_media_type) + return parse(content, context, probe=probe) + + +__all__ = [ + "DOCUMENT_SCHEMA_VERSION", "DocumentBlock", "DocumentLimits", + "DocumentMetadata", "DocumentParseContext", "DocumentParseWarning", + "DocumentProbe", "ParsedDocument", "UnsupportedContent", + "AmbiguousParser", "DocumentParser", "ParserEntry", "ParserRegistryError", + "list_parsers", "parse", "parse_bytes", "parser_names", "probe_bytes", + "select", +] diff --git a/openmind/documents/builder.py b/openmind/documents/builder.py new file mode 100644 index 0000000..4c4b8d3 --- /dev/null +++ b/openmind/documents/builder.py @@ -0,0 +1,191 @@ +"""Shared block assembly for every parser. + +Ten parsers each enforcing "cap the block count, cap the block length, downgrade +to partial, record the exact warning, keep the heading stack consistent" would be +ten chances to get one of those wrong, and the wrong ones would be the silent +ones. So it is written once here, and a parser only decides WHAT a block is. + +WHAT THE BUILDER GUARANTEES +--------------------------- +* Ordinals are dense and assigned in emission order. +* ``block_key`` is unique within the document (a deterministic ``#n`` suffix + resolves a collision — never a random id, because the key is hashed into the + Revision's ``structure_hash``). +* A block longer than ``max_block_chars`` is cut at the limit AND recorded as a + truncation warning naming the block, and the document becomes ``partial``. + Truncation is never silent. +* Once ``max_blocks`` is reached, further blocks are refused, the document + becomes ``partial``, and exactly ONE limit warning is emitted (not one per + refused block, which would turn a bounded document into an unbounded warning + list). +* The heading stack yields each block's ``heading_path`` and ``parent_key``, so + a parser never maintains that bookkeeping itself. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple + +from ..domain.types import ContentMode, DocumentBlockType +from .models import DocumentBlock, ParsedDocument + +#: Where a truncated block's text is cut. Kept as its own constant so the marker +#: is testable and identical everywhere. +TRUNCATION_MARKER = "\n[truncated by OpenMind: block exceeded the size limit]" + + +class DocumentBuilder: + """Accumulates blocks into a :class:`ParsedDocument` under the parse limits.""" + + def __init__(self, doc: ParsedDocument, *, max_blocks: int, + max_block_chars: int, root_key: str = "doc") -> None: + self.doc = doc + self.max_blocks = max(1, int(max_blocks)) + self.max_block_chars = max(1, int(max_block_chars)) + self.root_key = root_key + self._keys: Dict[str, int] = {} + self._limit_reported = False + #: (level, block_key, heading text) for each open heading, outermost first. + self._headings: List[Tuple[int, str, str]] = [] + + # -- state -------------------------------------------------------------- + @property + def count(self) -> int: + return len(self.doc.blocks) + + @property + def full(self) -> bool: + """Whether the block budget is spent — and, if so, RECORD that. + + Reading this is not a neutral query. Every parser consults it as its + early-exit guard, so the moment it first answers True is the exact moment + content starts being dropped. Reporting here (rather than only inside + :meth:`add`) is what stops a parser that breaks out of its loop from + producing a silently truncated document still labelled ``parsed``. + """ + if self.count >= self.max_blocks: + self._report_block_limit() + return True + return False + + @property + def heading_path(self) -> List[str]: + return [text for _, _, text in self._headings] + + @property + def current_parent(self) -> str: + """The innermost open heading's key, or the root.""" + return self._headings[-1][1] if self._headings else self.root_key + + def push_heading(self, level: int, key: str, text: str) -> None: + """Open a heading at *level*, closing any equal or deeper ones first. + + Called AFTER the heading's own block is added, so the heading block's own + ``heading_path`` names its ancestors and not itself. + """ + while self._headings and self._headings[-1][0] >= level: + self._headings.pop() + self._headings.append((int(level), key, text)) + + # -- emission ----------------------------------------------------------- + def add(self, block_type: str, text: str, *, key: str, + locator: Optional[Dict[str, Any]] = None, + content_mode: str = ContentMode.VERBATIM, + metadata: Optional[Dict[str, Any]] = None, + parent_key: Optional[str] = None, + heading_path: Optional[List[str]] = None, + indexable: Optional[bool] = None) -> Optional[DocumentBlock]: + """Append one block, or return None when the block budget is spent. + + ``indexable`` defaults to "content-bearing and not a structural + container", which is the rule the vector projection relies on. + """ + if self.full: + self._report_block_limit() + return None + + text = "" if text is None else str(text) + if len(text) > self.max_block_chars: + keep = max(0, self.max_block_chars - len(TRUNCATION_MARKER)) + text = text[:keep] + TRUNCATION_MARKER + self.doc.mark_partial( + "limit-exceeded", + f"block {key!r} exceeded DOCUMENT_MAX_BLOCK_CHARS " + f"({self.max_block_chars}) and was truncated", + limit="DOCUMENT_MAX_BLOCK_CHARS", block=key, + allowed=self.max_block_chars) + + if indexable is None: + indexable = (block_type not in DocumentBlockType.CONTAINERS + and bool(text.strip())) + + block = DocumentBlock( + block_key=self._unique(key), + block_type=block_type, + ordinal=self.count, + text=text, + parent_key=(self.current_parent if parent_key is None else parent_key), + heading_path=(list(self.heading_path) if heading_path is None + else list(heading_path)), + content_mode=content_mode, + locator=dict(locator or {}), + metadata=dict(metadata or {}), + indexable=bool(indexable)) + self.doc.blocks.append(block) + return block + + def add_root(self, title: str, locator: Dict[str, Any], *, + metadata: Optional[Dict[str, Any]] = None) -> DocumentBlock: + """The single ``document`` root block. Always stored, never indexed: it + is scaffolding, and embedding a title on its own dilutes retrieval.""" + block = DocumentBlock( + block_key=self.root_key, block_type=DocumentBlockType.DOCUMENT, + ordinal=self.count, text=title or "", parent_key="", + heading_path=[], content_mode=ContentMode.DERIVED, + locator=dict(locator), metadata=dict(metadata or {}), + indexable=False) + self.doc.blocks.append(block) + self._keys[self.root_key] = 1 + return block + + # -- internals ---------------------------------------------------------- + def _unique(self, key: str) -> str: + """A collision-free block key. Deterministic: the same input sequence + always produces the same keys, because the suffix counts prior uses of + that exact key rather than a global counter.""" + key = key or "block" + seen = self._keys.get(key, 0) + self._keys[key] = seen + 1 + return key if seen == 0 else f"{key}#{seen}" + + def _report_block_limit(self) -> None: + if self._limit_reported: + return + self._limit_reported = True + self.doc.mark_partial( + "limit-exceeded", + f"document reached DOCUMENT_MAX_BLOCKS ({self.max_blocks}); the " + f"remaining structure was not extracted", + limit="DOCUMENT_MAX_BLOCKS", allowed=self.max_blocks) + + +def slug(text: str, limit: int = 48) -> str: + """A short, deterministic, filesystem-safe fragment of *text*. + + Used inside block keys so a key reads as ``h2-interfaces-namecheck`` rather + than ``h2-7``. Lower-cased, non-alphanumerics collapsed to single hyphens. + """ + out: List[str] = [] + prev_dash = False + for ch in (text or "").lower(): + if ch.isalnum(): + out.append(ch) + prev_dash = False + elif not prev_dash and out: + out.append("-") + prev_dash = True + if len(out) >= limit: + break + return "".join(out).strip("-") + + +__all__ = ["DocumentBuilder", "TRUNCATION_MARKER", "slug"] diff --git a/openmind/documents/candidates.py b/openmind/documents/candidates.py new file mode 100644 index 0000000..dcbc7db --- /dev/null +++ b/openmind/documents/candidates.py @@ -0,0 +1,583 @@ +"""Deterministic candidate association between a document and existing knowledge. + +WHY "CANDIDATE" AND NOT "RELATION" +---------------------------------- +Appending a document to a workspace should immediately tell you what it might +touch. But "this requirement document mentions `NameCheckService`" and "this +requirement is implemented by `NameCheckService`" are completely different +claims, and only the first is observable without semantic verification. So this +module produces the first kind only. + +`implements`, `refines`, `verifies` and `contradicts` are deliberately absent +from :class:`~openmind.domain.types.CandidateType`. Every result carries +``status: "candidate"``, and **nothing here is persisted** — candidates are +recomputed on demand, so no unverified assertion can accumulate in the database +and later be mistaken for a fact. Canonical Relations are Phase 4. + +SIGNAL ORDER AND CONFIDENCE +--------------------------- +Deterministic signals run first and semantic retrieval runs last, because that +ordering is what keeps confidence honest: + + high an exact, explicit identifier matched a real target + (a workspace file path, a code symbol, a glossary term) + medium an exact match after deterministic normalization + (an API path + method, a configuration key, a database object) + low semantic retrieval only + +Nothing above ``low`` is ever produced by similarity. A candidate is only +emitted when its TARGET actually exists in the workspace — a document mentioning +`OrderService` when no such symbol was ever indexed produces nothing, because +there is no relationship to be a candidate for. +""" +from __future__ import annotations + +import re +from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple + +from ..domain.types import (CANDIDATE_STATUS, CandidateConfidence, CandidateType) + +#: Hard ceiling on returned candidates, whatever a caller asks for. +MAX_CANDIDATES = 100 + +#: Requirement-like ids: REQ-NC-017, ADR-12, BR-4711. +_REQUIREMENT_RE = re.compile(r"\b(?:REQ|ADR|BR|NFR|AC|US)-[A-Z0-9]{1,8}" + r"(?:-\d{1,6})?\b") +#: Change-request / ticket ids: ABC-1234. Deliberately requires >=2 letters and +#: a plain numeric tail, so it cannot swallow a requirement id. +_TICKET_RE = re.compile(r"\b[A-Z]{2,10}-\d{1,6}\b") +#: Error codes: NC-100, E-4021. +_ERROR_CODE_RE = re.compile(r"\b[A-Z]{1,5}-\d{2,5}\b") +#: Dotted configuration keys / topic names: namecheck.review.timeout.minutes +_CONFIG_KEY_RE = re.compile(r"\b[a-z][a-z0-9]*(?:\.[a-z0-9][a-z0-9_]*){2,}\b") +#: API path, optionally preceded by an HTTP method. +_API_RE = re.compile( + r"\b(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS)\b\s+(/[\w\-{}/]*)|" + r"(? None: + self.candidate_type = candidate_type + self.confidence = confidence + self.reason = reason + self.document_evidence = document_evidence + self.target = target + self.target_evidence = target_evidence or {} + self.retrieval_method = retrieval_method + self.mention = mention + + def key(self) -> Tuple[str, str, str, str]: + return (self.candidate_type, self.mention, + str(self.target.get("kind", "")), str(self.target.get("id", ""))) + + def as_dict(self) -> Dict[str, Any]: + return { + "candidate_type": self.candidate_type, + "confidence": self.confidence, + "reason": self.reason, + "mention": self.mention, + "document_evidence": dict(self.document_evidence), + "target": dict(self.target), + "target_evidence": dict(self.target_evidence), + "retrieval_method": self.retrieval_method, + # Never omitted. A consumer must not be able to read a candidate as + # a confirmed relation just because it forgot to check. + "status": CANDIDATE_STATUS, + } + + +_CONFIDENCE_RANK = {CandidateConfidence.HIGH: 0, CandidateConfidence.MEDIUM: 1, + CandidateConfidence.LOW: 2} + + +def find_candidates(workspace_id: str, asset_id: str, *, limit: int = 30, + repo: Any = None, searcher: Any = None) -> Dict[str, Any]: + """Deterministic candidates for one document Asset's CURRENT revision. + + Reads the document's stored Segments (never re-parses), extracts explicit + identifiers from their text, and resolves each against what the workspace + actually knows. Returns + ``{workspace_id, asset_id, revision_id, candidates, count, signals, + status}``. + """ + from .. import db as db_module + repository = repo if repo is not None else db_module + limit = max(1, min(int(limit or 30), MAX_CANDIDATES)) + + asset = repository.get_asset(workspace_id, asset_id) + if not asset: + return {"workspace_id": workspace_id, "asset_id": asset_id, + "revision_id": "", "candidates": [], "count": 0, + "signals": {}, "status": CANDIDATE_STATUS} + revision_id = asset.get("current_revision_id") or "" + blocks = _document_blocks(repository, workspace_id, revision_id) + + index = _WorkspaceIndex(repository, workspace_id, exclude_asset=asset_id) + found: Dict[Tuple[str, str, str, str], Candidate] = {} + signals: Dict[str, int] = {} + + for scan in (_scan_files, _scan_symbols, _scan_requirements, + _scan_code_identifiers, _scan_api, _scan_config_keys, + _scan_database_objects, _scan_glossary): + for candidate in scan(blocks, index): + key = candidate.key() + existing = found.get(key) + if existing is None or _better(candidate, existing): + found[key] = candidate + signals[candidate.retrieval_method] = \ + signals.get(candidate.retrieval_method, 0) + 1 + + deterministic = list(found.values()) + if len(deterministic) < limit: + for candidate in _scan_semantic(workspace_id, asset, blocks, index, + limit - len(deterministic), searcher): + key = candidate.key() + if key not in found: + found[key] = candidate + signals[candidate.retrieval_method] = \ + signals.get(candidate.retrieval_method, 0) + 1 + + ordered = sorted(found.values(), + key=lambda c: (_CONFIDENCE_RANK.get(c.confidence, 3), + c.candidate_type, c.mention)) + return { + "workspace_id": workspace_id, + "asset_id": asset_id, + "revision_id": revision_id, + "candidates": [c.as_dict() for c in ordered[:limit]], + "count": min(len(ordered), limit), + "total_found": len(ordered), + "signals": signals, + # Repeated at the top level so a caller reading only the envelope still + # sees it. + "status": CANDIDATE_STATUS, + "note": ("These are OBSERVED MENTIONS, not verified relationships. " + "No candidate asserts that the document implements, refines, " + "verifies or contradicts its target."), + } + + +def _better(new: Candidate, old: Candidate) -> bool: + return _CONFIDENCE_RANK.get(new.confidence, 3) < \ + _CONFIDENCE_RANK.get(old.confidence, 3) + + +def _document_blocks(repo: Any, workspace_id: str, + revision_id: str) -> List[Dict[str, Any]]: + """The document's stored segments with their evidence ids. Bounded.""" + if not revision_id: + return [] + segments = repo.list_segments(workspace_id, revision_id, limit=1000) + evidence = repo.evidence_ids_for_revision(workspace_id, revision_id) + out: List[Dict[str, Any]] = [] + for segment in segments: + excerpt = "" + record = repo.get_evidence_for_segment(workspace_id, segment["id"]) + if record: + excerpt = record.get("excerpt", "") + out.append({ + "segment_id": segment["id"], + "evidence_id": evidence.get(segment["id"], ""), + "block_type": segment["segment_type"], + "text": excerpt, + "locator": (record or {}).get("locator", {}), + }) + return out + + +class _WorkspaceIndex: + """What the workspace already knows, loaded once. + + Every lookup is an EXACT match against something that genuinely exists. A + mention with no real target produces no candidate — an unresolvable + "candidate" is noise, not a finding. + """ + + def __init__(self, repo: Any, workspace_id: str, + exclude_asset: str = "") -> None: + self.repo = repo + self.workspace_id = workspace_id + self.exclude_asset = exclude_asset + self._assets: Dict[str, Dict[str, Any]] = {} + self._symbols: Optional[Dict[str, Dict[str, Any]]] = None + self._glossary: Optional[Dict[str, Dict[str, Any]]] = None + self._db_objects: Optional[Dict[str, Dict[str, Any]]] = None + for key, record in repo.list_asset_index(workspace_id).items(): + if record.get("asset_id") != exclude_asset: + self._assets[key] = record + + # -- files ---------------------------------------------------------- + def file(self, path: str) -> Optional[Dict[str, Any]]: + record = self._assets.get(path) + if record: + return {"kind": "file", "id": record["asset_id"], + "logical_key": path} + # A document usually cites a bare filename, not a full workspace path. + # A UNIQUE basename match is still exact; an ambiguous one is not a + # match at all, because naming the wrong file is worse than no candidate. + matches = [k for k in self._assets if k.rsplit("/", 1)[-1] == path] + if len(matches) == 1: + return {"kind": "file", "id": self._assets[matches[0]]["asset_id"], + "logical_key": matches[0]} + return None + + # -- symbols -------------------------------------------------------- + @property + def symbols(self) -> Dict[str, Dict[str, Any]]: + if self._symbols is None: + self._symbols = {} + for name, record in self.repo.list_workspace_symbols( + self.workspace_id, exclude_asset=self.exclude_asset).items(): + self._symbols[name] = record + return self._symbols + + def symbol(self, name: str) -> Optional[Dict[str, Any]]: + return self.symbols.get(name) + + # -- glossary ------------------------------------------------------- + @property + def glossary(self) -> Dict[str, Dict[str, Any]]: + if self._glossary is None: + self._glossary = {} + try: + from .. import glossary as glossary_module, mapio + document = mapio.load_glossary(self.workspace_id) + for entry in (document or {}).get("terms", []) or []: + term = str(entry.get("term") or "").strip() + if term: + self._glossary[term] = entry + except Exception: + self._glossary = {} + return self._glossary + + # -- database objects ------------------------------------------------ + @property + def database_objects(self) -> Dict[str, Dict[str, Any]]: + """Table and view names, taken ONLY from ``sql-object`` segments. + + That restriction matters. A ``.sql`` file ingested through the CODE + pipeline is segmented into generic line ranges whose symbol is the + filename, so accepting any symbol from a ``database-schema`` asset would + make the word "sql" a database object and match it in every document. + A real object name exists only where the SQL parser structured one. + """ + if self._db_objects is None: + self._db_objects = {} + for name, record in self.symbols.items(): + if record.get("segment_type") != "sql-object": + continue + bare = name.rsplit(".", 1)[-1].strip() + if len(bare) >= 3: + self._db_objects.setdefault(bare.lower(), record) + return self._db_objects + + +def _mentions(blocks: Sequence[Dict[str, Any]], pattern: re.Pattern, + group: int = 0) -> Iterable[Tuple[str, Dict[str, Any]]]: + """Yield ``(mention, block)`` for each distinct pattern match, bounded.""" + seen: Set[str] = set() + emitted = 0 + for block in blocks: + for match in pattern.finditer(block.get("text") or ""): + value = (match.group(group) or "").strip() + if not value or value in seen: + continue + seen.add(value) + emitted += 1 + yield value, block + if emitted >= _MAX_MENTIONS_PER_SIGNAL: + return + + +def _document_evidence(block: Dict[str, Any]) -> Dict[str, Any]: + return {"segment_id": block.get("segment_id", ""), + "evidence_id": block.get("evidence_id", ""), + "block_type": block.get("block_type", ""), + "locator": dict(block.get("locator") or {})} + + +# --------------------------------------------------------------------------- +# Signals +# --------------------------------------------------------------------------- +def _scan_files(blocks, index) -> Iterable[Candidate]: + for mention, block in _mentions(blocks, _FILE_RE): + target = index.file(mention) + if target is None: + continue + yield Candidate( + CandidateType.MENTIONS_FILE, CandidateConfidence.HIGH, + f"the document names {mention!r}, which is an indexed workspace file", + _document_evidence(block), target, "exact-file-path", mention) + + +def _scan_symbols(blocks, index) -> Iterable[Candidate]: + for mention, block in _mentions(blocks, _SYMBOL_RE): + target = index.symbol(mention) + if target is None: + continue + yield Candidate( + CandidateType.MENTIONS_SYMBOL, CandidateConfidence.HIGH, + f"the document names {mention!r}, which is an indexed code symbol", + _document_evidence(block), + {"kind": "symbol", "id": target.get("segment_id", ""), + "symbol": mention, "asset_id": target.get("asset_id", ""), + "logical_key": target.get("logical_key", "")}, + "exact-symbol", mention, + {"segment_id": target.get("segment_id", ""), + "evidence_id": target.get("evidence_id", "")}) + + +def _scan_requirements(blocks, index) -> Iterable[Candidate]: + """Requirement-like ids shared with ANOTHER document. + + A requirement id that appears only in this document connects it to nothing, + so nothing is emitted for it. The finding is the SHARED id — a test case and + a specification naming the same requirement. + """ + for mention, block in _mentions(blocks, _REQUIREMENT_RE): + for target in _documents_mentioning(index, mention): + yield Candidate( + CandidateType.MENTIONS_DOCUMENT, CandidateConfidence.HIGH, + f"both documents contain the identifier {mention!r}", + _document_evidence(block), target, "exact-requirement-id", + mention, {"evidence_id": target.pop("_evidence_id", "")}) + + +def _classify_code(mention: str) -> Optional[Tuple[str, str, str]]: + """``(candidate_type, retrieval_method, noun)`` for a ``LETTERS-DIGITS`` id. + + ``REQ-NC-017``, ``ABC-1234`` and ``NC-100`` all share one lexical shape, so a + mention must be classified ONCE and by a stated rule — otherwise the same + ``NC-100`` comes back as both a ticket and an error code, which reads like + two findings when there is one. + + The rule, in order: + + 1. a known requirement prefix (REQ/ADR/BR/NFR/AC/US) -> requirement; + 2. a project-key shape — at least three letters AND at least four digits, + the JIRA convention -> ticket; + 3. otherwise -> error code. + + Rules 2 and 3 are a CONVENTION, not a certainty; the emitted reason says so + rather than asserting the interpretation. + """ + if _REQUIREMENT_RE.fullmatch(mention): + return None # handled by _scan_requirements + head, _, tail = mention.partition("-") + if len(head) >= 3 and len(tail) >= 4 and tail.isdigit(): + return (CandidateType.MENTIONS_DOCUMENT, "exact-ticket-id", + "change-request or ticket identifier") + return (CandidateType.MENTIONS_CONFIGURATION, "normalized-error-code", + "error code") + + +def _scan_code_identifiers(blocks, index) -> Iterable[Candidate]: + """Tickets and error codes — one classification per mention (see + :func:`_classify_code`).""" + seen: Set[str] = set() + for pattern in (_TICKET_RE, _ERROR_CODE_RE): + for mention, block in _mentions(blocks, pattern): + if mention in seen: + continue + seen.add(mention) + classified = _classify_code(mention) + if classified is None: + continue + candidate_type, method, noun = classified + confidence = (CandidateConfidence.HIGH + if method == "exact-ticket-id" + else CandidateConfidence.MEDIUM) + for target in _documents_mentioning(index, mention): + yield Candidate( + candidate_type, confidence, + f"both documents contain {mention!r}, which matches the " + f"{noun} convention", + _document_evidence(block), target, method, mention, + {"evidence_id": target.pop("_evidence_id", "")}) + + +def _scan_api(blocks, index) -> Iterable[Candidate]: + seen: Set[str] = set() + emitted = 0 + for block in blocks: + for match in _API_RE.finditer(block.get("text") or ""): + method = (match.group(1) or "").upper() + path = (match.group(2) or match.group(3) or "").strip() + if not path or len(path) < 2: + continue + mention = f"{method} {path}".strip() + if mention in seen: + continue + seen.add(mention) + for target in _documents_mentioning(index, path): + emitted += 1 + yield Candidate( + CandidateType.MENTIONS_API, CandidateConfidence.MEDIUM, + f"both documents describe the API path {path!r}", + _document_evidence(block), target, "normalized-api-path", + mention, {"evidence_id": target.pop("_evidence_id", "")}) + if emitted >= _MAX_MENTIONS_PER_SIGNAL: + return + + +def _scan_config_keys(blocks, index) -> Iterable[Candidate]: + """Dotted keys: configuration properties and message/topic names alike. + + They are the same lexical shape and the same kind of observation, so they + share a signal rather than being split by a guess about which one a given + key is. + """ + for mention, block in _mentions(blocks, _CONFIG_KEY_RE): + for target in _documents_mentioning(index, mention): + yield Candidate( + CandidateType.MENTIONS_CONFIGURATION, CandidateConfidence.MEDIUM, + f"both documents reference the key or topic {mention!r}", + _document_evidence(block), target, "normalized-config-key", + mention, {"evidence_id": target.pop("_evidence_id", "")}) + symbol = index.symbol(mention) + if symbol is not None: + yield Candidate( + CandidateType.MENTIONS_CONFIGURATION, CandidateConfidence.MEDIUM, + f"the document references {mention!r}, which appears in " + f"indexed configuration", + _document_evidence(block), + {"kind": "configuration", "id": symbol.get("segment_id", ""), + "symbol": mention, "asset_id": symbol.get("asset_id", ""), + "logical_key": symbol.get("logical_key", "")}, + "normalized-config-key", mention) + + +def _scan_database_objects(blocks, index) -> Iterable[Candidate]: + objects = index.database_objects + if not objects: + return + seen: Set[str] = set() + for block in blocks: + for word in re.findall(r"\b[a-z][a-z0-9_]{2,}\b", + (block.get("text") or "").lower()): + if word in seen or word not in objects: + continue + seen.add(word) + target = objects[word] + yield Candidate( + CandidateType.MENTIONS_DATABASE_OBJECT, + CandidateConfidence.MEDIUM, + f"the document names {word!r}, which is a database object in " + f"the indexed schema", + _document_evidence(block), + {"kind": "database-object", "id": target.get("segment_id", ""), + "symbol": word, "asset_id": target.get("asset_id", ""), + "logical_key": target.get("logical_key", "")}, + "exact-database-object", word) + if len(seen) >= _MAX_MENTIONS_PER_SIGNAL: + return + + +def _scan_glossary(blocks, index) -> Iterable[Candidate]: + terms = index.glossary + if not terms: + return + seen: Set[str] = set() + for block in blocks: + text = block.get("text") or "" + for term, entry in terms.items(): + if term in seen or len(term) < 3: + continue + from .. import tokenmatch + if tokenmatch.match_kind(text, term, case_sensitive=True) != "token": + continue + seen.add(term) + yield Candidate( + CandidateType.MENTIONS_DOCUMENT, CandidateConfidence.HIGH, + f"the document uses the glossary term {term!r}", + _document_evidence(block), + {"kind": "glossary-term", "id": term, "symbol": term, + "logical_key": str(entry.get("source_file") or "")}, + "exact-glossary-term", term) + if len(seen) >= _MAX_MENTIONS_PER_SIGNAL: + return + + +def _documents_mentioning(index: _WorkspaceIndex, + token: str) -> Iterable[Dict[str, Any]]: + """Other DOCUMENTS in the workspace whose indexed blocks contain *token* as a + complete token. Uses the document collection, so it costs one bounded query + rather than a scan of every stored segment.""" + from .. import document_rag + try: + result = document_rag.search(index.workspace_id, token, limit=5) + except Exception: + return + for hit in result.get("hits", []): + if hit.get("asset_id") == index.exclude_asset: + continue + if "exact-identifier" not in (hit.get("retrieval_sources") or []): + continue + yield {"kind": "document", "id": hit.get("asset_id", ""), + "logical_key": hit.get("logical_key", ""), + "title": hit.get("title", ""), + "segment_id": hit.get("segment_id", ""), + "_evidence_id": hit.get("evidence_id", "")} + + +def _scan_semantic(workspace_id: str, asset: Dict[str, Any], + blocks: Sequence[Dict[str, Any]], index: _WorkspaceIndex, + budget: int, searcher: Any) -> Iterable[Candidate]: + """The fallback leg: purely embedding-similar code, labelled LOW. + + Runs last and only fills the remaining budget, so a deterministic finding is + never displaced by a similarity guess. It is the ONE signal allowed to + produce a candidate without an exact match, and its confidence says so. + """ + if budget <= 0 or not blocks: + return + query = " ".join((b.get("text") or "")[:200] for b in blocks[:3]).strip() + if not query: + return + search = searcher + if search is None: + from .. import rag + + def search(q: str) -> List[Dict[str, Any]]: + return rag.retrieve([workspace_id], q, k=budget).get( + "code_chunks", []) + try: + results = search(query) + except Exception: + return + for chunk in list(results)[:budget]: + path = chunk.get("file_path") or "" + if not path: + continue + yield Candidate( + CandidateType.SIMILAR_CONTENT, CandidateConfidence.LOW, + "retrieved by embedding similarity only; no explicit identifier " + "in the document matches this code", + {"segment_id": blocks[0].get("segment_id", ""), + "evidence_id": blocks[0].get("evidence_id", ""), + "block_type": blocks[0].get("block_type", ""), + "locator": dict(blocks[0].get("locator") or {})}, + {"kind": "code", "id": chunk.get("id", ""), "logical_key": path, + "symbol": chunk.get("symbol") or ""}, + "semantic-retrieval", path) + + +__all__ = ["MAX_CANDIDATES", "Candidate", "find_candidates"] diff --git a/openmind/documents/csv_parser.py b/openmind/documents/csv_parser.py new file mode 100644 index 0000000..c2778da --- /dev/null +++ b/openmind/documents/csv_parser.py @@ -0,0 +1,236 @@ +"""CSV and TSV, using the standard-library reader. + +WHY THE STDLIB +-------------- +``csv`` implements RFC 4180 quoting, embedded newlines and escape handling +correctly, has no dependencies, and cannot be talked into evaluating anything. +Dialect detection uses ``csv.Sniffer`` on a BOUNDED sample — an unbounded sniff +on a hostile file is a denial-of-service in one line of code. + +MALFORMED INPUT IS REPORTED, NOT REINTERPRETED +---------------------------------------------- +Two things a naive importer does silently, and this one refuses to: + +* **Guessing a dialect it could not detect.** When the sniffer fails, the parser + falls back to the extension's conventional delimiter AND records a + ``dialect-uncertain`` warning, so a mis-split file is visible rather than + mysterious. +* **Reshaping ragged rows.** A row whose field count differs from the header's + is kept as-is, counted, and reported in a ``ragged-rows`` warning. Padding or + truncating it to fit would fabricate data. + +Header detection is deterministic: the first row is the header when every one of +its fields is non-empty and none of them parses as a number. Anything less +certain is treated as data, and the columns are named ``col1..colN``. +""" +from __future__ import annotations + +import csv +import io +import os +from typing import Any, Dict, List, Optional + +from ..domain.types import ContentMode, DocumentBlockType +from .builder import DocumentBuilder +from .models import DocumentParseContext, DocumentProbe, ParsedDocument +from .security import decode_text + +#: Bounded sniff window. Big enough to see a representative sample, small enough +#: that sniffing a hostile 25 MB file is still instant. +_SNIFF_BYTES = 16_384 +_EXTENSIONS = frozenset({".csv", ".tsv"}) +_DEFAULT_DELIMITER = {".csv": ",", ".tsv": "\t"} + +#: Column-name limit for a header row. A "CSV" with 10k columns is a +#: transposed data dump, not a document. +_MAX_COLUMNS = 512 + + +def CLAIMS(probe: DocumentProbe) -> bool: # noqa: N802 - registry protocol + if probe.magic.get("binary") or probe.magic.get("zip") \ + or probe.magic.get("pdf"): + return False + return probe.extension in _EXTENSIONS + + +class CsvParser: + """Row-oriented CSV/TSV extraction with bounded dialect detection.""" + + name = "csv" + version = "1.0" + + def supports(self, probe: DocumentProbe) -> bool: + return CLAIMS(probe) + + def parse(self, content: bytes, + context: DocumentParseContext) -> ParsedDocument: + text, encoding, lossy = decode_text(content) + ext = os.path.splitext(context.logical_key)[1].lower() + doc = ParsedDocument( + parser_name=self.name, parser_version=self.version, + media_type=("text/tab-separated-values" if ext == ".tsv" + else "text/csv"), + title=context.filename or context.logical_key) + if lossy: + doc.add_warning( + "decode-fallback", + "the file is not valid UTF-8; it was decoded with replacement " + "characters, so some values may not be exact", encoding=encoding) + doc.metadata.extra["encoding"] = encoding + + delimiter, sniffed = _detect_delimiter(text, ext, doc) + doc.metadata.extra["delimiter"] = ("\\t" if delimiter == "\t" + else delimiter) + doc.metadata.extra["dialect_detected"] = sniffed + + builder = DocumentBuilder(doc, max_blocks=context.limits.max_blocks, + max_block_chars=context.limits.max_block_chars) + key = context.logical_key + builder.add_root(doc.title, _locator(key, "A1")) + + reader = csv.reader(io.StringIO(text, newline=""), delimiter=delimiter) + try: + rows = _read_bounded(reader, context.limits.csv_max_rows, doc) + except csv.Error as exc: + # A structurally broken file (an unterminated quote spanning the + # whole document) is reported as partial with whatever was read, + # never silently re-parsed with different rules. + doc.mark_partial("malformed-csv", + f"the file could not be read past a CSV error: {exc}") + rows = [] + + if not rows: + doc.coverage["rows"] = 0 + doc.coverage["columns"] = 0 + return doc + + headers, data_rows, header_row_no = _split_header(rows) + doc.metadata.extra["has_header"] = header_row_no == 1 + table_key = "table-0001" + builder.add(DocumentBlockType.TABLE, " | ".join(headers), key=table_key, + locator=_locator(key, _range(1, len(headers))), + content_mode=ContentMode.DERIVED, indexable=False, + metadata={"columns": len(headers), "rows": len(data_rows), + "headers": ", ".join(headers[:_MAX_COLUMNS])}) + + ragged = 0 + for offset, cells in enumerate(data_rows): + row_no = header_row_no + offset + 1 + if len(cells) != len(headers): + ragged += 1 + pairs = [f"{headers[c] if c < len(headers) else f'col{c + 1}'}: {v}" + for c, v in enumerate(cells) if str(v).strip()] + if not pairs: + continue + builder.add( + DocumentBlockType.TABLE_ROW, "; ".join(pairs), + key=f"{table_key}-r{row_no:06d}", + locator=_locator(key, _range(row_no, len(cells)), + row_index=row_no), + # DERIVED: `header: value` is a rendering of the row, not the + # file's literal bytes. The locator still cites the exact row. + content_mode=ContentMode.DERIVED, parent_key=table_key, + metadata={"row": row_no, "cells": len(cells)}) + if builder.full: + break + + if ragged: + doc.add_warning( + "ragged-rows", + f"{ragged} row(s) do not have the header's {len(headers)} " + f"field(s); they were kept exactly as read, not padded or " + f"truncated", + rows=ragged, expected=len(headers)) + doc.coverage["rows"] = len(data_rows) + doc.coverage["columns"] = len(headers) + return doc + + +def _detect_delimiter(text: str, ext: str, doc: ParsedDocument) -> tuple: + """(delimiter, was_sniffed). Falls back to the extension's convention and + says so, rather than guessing silently.""" + sample = text[:_SNIFF_BYTES] + if sample.strip(): + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t|") + return dialect.delimiter, True + except csv.Error: + pass + fallback = _DEFAULT_DELIMITER.get(ext, ",") + doc.add_warning( + "dialect-uncertain", + f"the CSV dialect could not be detected from the first " + f"{_SNIFF_BYTES} bytes; the conventional delimiter for {ext or 'csv'} " + f"({'tab' if fallback == chr(9) else fallback!r}) was used", + delimiter=("\\t" if fallback == "\t" else fallback)) + return fallback, False + + +def _read_bounded(reader: Any, max_rows: int, + doc: ParsedDocument) -> List[List[str]]: + rows: List[List[str]] = [] + for row in reader: + if len(rows) >= max_rows: + doc.mark_partial( + "limit-exceeded", + f"the file has more than CSV_MAX_ROWS ({max_rows}) rows; the " + f"remainder was not read", + limit="CSV_MAX_ROWS", allowed=max_rows) + break + if any(str(c).strip() for c in row): + rows.append([str(c) for c in row]) + return rows + + +def _looks_numeric(value: str) -> bool: + v = (value or "").strip().replace(",", "").replace("%", "") + if not v: + return False + try: + float(v) + return True + except ValueError: + return False + + +def _split_header(rows: List[List[str]]) -> tuple: + """(headers, data_rows, header_row_number). + + The first row is the header only when every field is non-empty and none is + numeric. Otherwise the data starts at row 1 and columns are positional — + inventing header names from a data row would mislabel every citation. + """ + first = rows[0] + is_header = (len(first) > 0 + and all(str(c).strip() for c in first) + and not any(_looks_numeric(c) for c in first)) + if is_header: + return ([str(c).strip() for c in first[:_MAX_COLUMNS]], rows[1:], 1) + width = max(len(r) for r in rows) + return ([f"col{i + 1}" for i in range(min(width, _MAX_COLUMNS))], rows, 0) + + +def _column_letter(index: int) -> str: + """1-based column index -> spreadsheet letter (1 -> A, 27 -> AA).""" + letters = "" + while index > 0: + index, rem = divmod(index - 1, 26) + letters = chr(ord("A") + rem) + letters + return letters or "A" + + +def _range(row_no: int, width: int) -> str: + last = _column_letter(max(1, width)) + return f"A{row_no}:{last}{row_no}" + + +def _locator(document: str, cell_range: str, + row_index: Optional[int] = None) -> Dict[str, Any]: + loc: Dict[str, Any] = {"kind": "spreadsheet-range", "document": document, + "sheet": "", "range": cell_range} + if row_index is not None: + loc["rowIndex"] = row_index + return loc + + +__all__ = ["CsvParser", "CLAIMS"] diff --git a/openmind/documents/docx_parser.py b/openmind/documents/docx_parser.py new file mode 100644 index 0000000..21c20ff --- /dev/null +++ b/openmind/documents/docx_parser.py @@ -0,0 +1,330 @@ +"""DOCX (Office Open XML wordprocessing) via ``python-docx``. + +SAFETY BEFORE PARSING +--------------------- +A DOCX is a ZIP of XML. Both halves are hostile-input surfaces, so both are +bounded before ``python-docx`` sees a single byte: + +1. :func:`~openmind.documents.security.inspect_zip` validates the package — + member count, per-member size, total expansion, compression ratio and path + traversal. Nothing is ever extracted to a filesystem path; the archive is + read from memory. +2. :func:`~openmind.documents.security.harden_xml` applies + ``defusedxml.defuse_stdlib()`` so the stdlib XML parsers ``python-docx`` uses + refuse external entities, external DTDs and entity-expansion bombs. If + ``defusedxml`` is not installed this parser declines the document rather than + parsing it unsafely — an unsafe parse is never the fallback. + +Macros are never executed, external links are never followed, and embedded +images are recorded as unsupported content rather than silently dropped. + +STRUCTURE +--------- +Heading level comes from the paragraph's STYLE (``Heading 1``…``Heading 9``, +``Title``), which is what the author actually asserted — never from font size or +from prose that looks like a heading. Body order is recovered by walking the +document body's XML children, so a table between two paragraphs stays between +them instead of being appended after every paragraph, which is what iterating +``document.paragraphs`` then ``document.tables`` would produce. +""" +from __future__ import annotations + +import io +import re +from typing import Any, Dict, Iterator, List, Optional, Tuple + +from ..domain.types import ContentMode, DocumentBlockType, DocumentParseStatus +from .builder import DocumentBuilder, slug +from .models import (DocumentParseContext, DocumentProbe, ParsedDocument, + dependency_unavailable) +from .probe import _DOCX_MARKER +from .security import DocumentSecurityError, harden_xml, inspect_zip + +_MEDIA = ("application/vnd.openxmlformats-officedocument" + ".wordprocessingml.document") + +_HEADING_STYLE_RE = re.compile(r"^heading\s*(\d)$", re.IGNORECASE) +_CODE_STYLE_RE = re.compile(r"(code|source|listing|monospace|preformatted)", + re.IGNORECASE) +_LIST_STYLE_RE = re.compile(r"(list|bullet)", re.IGNORECASE) + +#: OOXML namespace for wordprocessing elements, used for the body walk. +_W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" + + +def CLAIMS(probe: DocumentProbe) -> bool: # noqa: N802 - registry protocol + """A DOCX is claimed on PACKAGE STRUCTURE, never on the suffix. + + A renamed ZIP with a ``.docx`` extension does not contain + ``word/document.xml`` and is therefore not claimed — it falls through to be + reported as unsupported, which is the honest answer. + """ + return probe.is_zip_package and probe.has_zip_member(_DOCX_MARKER) + + +class DocxParser: + """Paragraphs, heading hierarchy, lists and tables from a DOCX package.""" + + name = "docx" + version = "1.0" + + def supports(self, probe: DocumentProbe) -> bool: + return CLAIMS(probe) + + def parse(self, content: bytes, + context: DocumentParseContext) -> ParsedDocument: + limits = context.limits + doc = ParsedDocument(parser_name=self.name, parser_version=self.version, + media_type=_MEDIA, + title=context.filename or context.logical_key) + + if not harden_xml(): + return dependency_unavailable(context.filename, self.name, + "defusedxml") + try: + import docx # python-docx + except ImportError: + return dependency_unavailable(context.filename, self.name, + "python-docx") + + try: + package = inspect_zip( + content, max_members=limits.zip_max_members, + max_total_bytes=limits.zip_max_total_bytes, + max_member_bytes=limits.zip_max_member_bytes, + max_ratio=limits.zip_max_ratio) + except DocumentSecurityError as exc: + doc.status = DocumentParseStatus.UNSUPPORTED + doc.reason = exc.code + doc.add_warning(exc.code, exc.message, **exc.detail) + return doc + + images = sum(1 for m in package["members"] if m.startswith("word/media/")) + if images: + doc.note_unsupported( + "embedded-image", images, + "embedded images are recorded but not extracted; OCR is not " + "performed in this phase") + macros = sum(1 for m in package["members"] + if m.startswith("word/vbaProject")) + if macros: + doc.note_unsupported("macro", macros, + "macros are never read or executed") + + document = docx.Document(io.BytesIO(content)) + _read_core_properties(document, doc) + + builder = DocumentBuilder(doc, max_blocks=limits.max_blocks, + max_block_chars=limits.max_block_chars) + key = context.logical_key + builder.add_root(doc.title, {"kind": "docx-paragraph", "document": key, + "paragraphIndex": 0}) + _walk_body(document, builder, key, doc, limits) + + for block in doc.blocks: + if (block.block_type == DocumentBlockType.HEADING + and block.metadata.get("level", 9) <= 1): + doc.title = block.text.strip() or doc.title + doc.blocks[0].text = doc.title + break + return doc + + +def _read_core_properties(document: Any, doc: ParsedDocument) -> None: + """Copy the OOXML core properties. Never raises — a document with a damaged + or absent core-properties part is still perfectly parseable.""" + try: + props = document.core_properties + except Exception: + return + def _text(value: Any) -> str: + if value is None: + return "" + return value.isoformat() if hasattr(value, "isoformat") else str(value) + + doc.metadata.author = _text(getattr(props, "author", "")) + doc.metadata.created = _text(getattr(props, "created", None)) + doc.metadata.modified = _text(getattr(props, "modified", None)) + doc.metadata.language = _text(getattr(props, "language", "")) + # `revision` is an explicit, documented DOCX field — the ONLY place a + # version label may come from here. Nothing is inferred from prose. + revision = getattr(props, "revision", None) + if revision: + doc.metadata.version_label = str(revision) + title = _text(getattr(props, "title", "")) + if title.strip(): + doc.title = title.strip() + subject = _text(getattr(props, "subject", "")) + if subject: + doc.metadata.extra["subject"] = subject + category = _text(getattr(props, "category", "")) + if category: + doc.metadata.extra["category"] = category + + +def _iter_body(document: Any) -> Iterator[Tuple[str, Any]]: + """Yield ``("paragraph"|"table", element)`` in true document order. + + ``document.paragraphs`` and ``document.tables`` are two separate flat lists, + so using them loses the interleaving — every table would land after every + paragraph and a table's caption would end up nowhere near it. Walking the + body's XML children preserves the order the author wrote. + """ + from docx.table import Table + from docx.text.paragraph import Paragraph + + body = document.element.body + for child in body.iterchildren(): + if child.tag == f"{_W}p": + yield "paragraph", Paragraph(child, document) + elif child.tag == f"{_W}tbl": + yield "table", Table(child, document) + + +def _style_name(paragraph: Any) -> str: + try: + return (paragraph.style.name or "").strip() + except Exception: + return "" + + +def _heading_level(style: str) -> Optional[int]: + if style.lower() == "title": + return 1 + match = _HEADING_STYLE_RE.match(style) + return int(match.group(1)) if match else None + + +def _column_letter(index: int) -> str: + letters = "" + while index > 0: + index, rem = divmod(index - 1, 26) + letters = chr(ord("A") + rem) + letters + return letters or "A" + + +def _walk_body(document: Any, builder: DocumentBuilder, key: str, + doc: ParsedDocument, limits: Any) -> None: + paragraph_index = 0 + table_index = 0 + truncated_paragraphs = False + truncated_tables = False + + for kind, element in _iter_body(document): + if builder.full: + break + if kind == "paragraph": + paragraph_index += 1 + if paragraph_index > limits.docx_max_paragraphs: + if not truncated_paragraphs: + truncated_paragraphs = True + doc.mark_partial( + "limit-exceeded", + f"the document has more than DOCX_MAX_PARAGRAPHS " + f"({limits.docx_max_paragraphs}) paragraphs; the " + f"remainder was not extracted", + limit="DOCX_MAX_PARAGRAPHS", + allowed=limits.docx_max_paragraphs) + continue + _emit_paragraph(builder, key, element, paragraph_index) + else: + table_index += 1 + if table_index > limits.docx_max_tables: + if not truncated_tables: + truncated_tables = True + doc.mark_partial( + "limit-exceeded", + f"the document has more than DOCX_MAX_TABLES " + f"({limits.docx_max_tables}) tables; the remainder was " + f"not extracted", + limit="DOCX_MAX_TABLES", allowed=limits.docx_max_tables) + continue + _emit_table(builder, key, element, table_index) + + doc.coverage["paragraphs"] = paragraph_index + doc.coverage["tables"] = table_index + + +def _emit_paragraph(builder: DocumentBuilder, key: str, paragraph: Any, + index: int) -> None: + text = (paragraph.text or "").strip() + if not text: + return + style = _style_name(paragraph) + locator: Dict[str, Any] = {"kind": "docx-paragraph", "document": key, + "paragraphIndex": index} + level = _heading_level(style) + if level is not None: + locator["headingPath"] = list(builder.heading_path) + block = builder.add( + DocumentBlockType.HEADING, text, + key=f"h{level}-{slug(text) or index}", locator=locator, + metadata={"level": level, "style": style}) + if block is not None: + builder.push_heading(level, block.block_key, text) + return + + locator["headingPath"] = list(builder.heading_path) + if _CODE_STYLE_RE.search(style): + block_type = DocumentBlockType.CODE_BLOCK + elif _LIST_STYLE_RE.search(style): + block_type = DocumentBlockType.LIST_ITEM + else: + block_type = DocumentBlockType.PARAGRAPH + builder.add(block_type, text, key=f"p-{index:06d}", locator=locator, + metadata={"style": style} if style else {}) + + +def _emit_table(builder: DocumentBuilder, key: str, table: Any, + table_index: int) -> None: + rows = list(table.rows) + header: List[str] = [] + if rows: + header = [(_cell_text(c)) for c in rows[0].cells] + + table_key = f"table-{table_index:04d}" + builder.add( + DocumentBlockType.TABLE, " | ".join(header), key=table_key, + locator={"kind": "docx-table-row", "document": key, + "tableIndex": table_index, "rowIndex": 0, + "cellRange": f"A1:{_column_letter(max(1, len(header)))}1"}, + content_mode=ContentMode.DERIVED, indexable=False, + metadata={"tableIndex": table_index, "rows": len(rows), + "columns": len(header), "headers": ", ".join(header)}) + + for row_no, row in enumerate(rows, start=1): + if builder.full: + break + cells = [_cell_text(c) for c in row.cells] + if not any(c.strip() for c in cells): + continue + # The header row is emitted too (it is real content someone may search + # for), but labelled positionally — pairing it with itself would produce + # the useless "Requirement: Requirement". + names = ([_column_letter(i + 1) for i in range(len(cells))] + if row_no == 1 else header) + pairs = [f"{names[i] if i < len(names) and names[i] else f'col{i + 1}'}" + f": {value}" + for i, value in enumerate(cells) if value.strip()] + builder.add( + DocumentBlockType.TABLE_ROW, "; ".join(pairs), + key=f"{table_key}-r{row_no:04d}", + locator={"kind": "docx-table-row", "document": key, + "tableIndex": table_index, "rowIndex": row_no, + "cellRange": f"A{row_no}:" + f"{_column_letter(max(1, len(cells)))}{row_no}"}, + # DERIVED: `header: value` is a rendering of the row, not the + # document's literal text. The locator still cites the exact row. + content_mode=ContentMode.DERIVED, parent_key=table_key, + metadata={"row": row_no, "cells": len(cells), + "header_row": row_no == 1}) + + +def _cell_text(cell: Any) -> str: + try: + return " ".join((cell.text or "").split()) + except Exception: + return "" + + +__all__ = ["DocxParser", "CLAIMS"] diff --git a/openmind/documents/html_parser.py b/openmind/documents/html_parser.py new file mode 100644 index 0000000..d3fe425 --- /dev/null +++ b/openmind/documents/html_parser.py @@ -0,0 +1,346 @@ +"""HTML structure extraction, using the standard library only. + +WHY THE STDLIB PARSER +--------------------- +``html.parser.HTMLParser`` is a tolerant, streaming, pure-Python tokenizer with +no network stack, no CSS engine and no JavaScript. That is exactly the right +capability envelope for untrusted input: there is nothing in it that *can* fetch +a linked resource or run a script, so "do not execute JavaScript, do not fetch +linked resources" is guaranteed by construction rather than by discipline. +Adding a heavier parser would buy fidelity we do not need and an attack surface +we would then have to bound. + +WHAT IS DROPPED, ON PURPOSE +--------------------------- +`` + +

real content

+ + + +""" +html = parse_bytes(hostile, logical_key="documents/h.html", filename="h.html") +body = " ".join(b.text for b in html.blocks) +check("html: script content is not indexed", "evil.invalid" not in body) +check("html: the cookie-stealing script text is absent", + "document.cookie" not in body) +check("html: style content is not indexed", "background" not in body) +check("html: hidden content is not indexed", "hidden secret" not in body) +check("html: noscript content is not indexed", "noscript secret" not in body) +check("html: real content IS indexed", "real content" in body) +check("html: an image is recorded, not fetched", + any(u.kind == "image" for u in html.unsupported_content)) + +# --------------------------------------------------------------------------- +# 7. Remote references are never fetched +# --------------------------------------------------------------------------- +remote_schema = b"""{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "R", "type": "object", + "properties": {"a": {"$ref": "https://example.invalid/remote.json"}}, + "$defs": {"B": {"type": "string"}} +}""" +schema = parse_bytes(remote_schema, logical_key="documents/r.json", + filename="r.json") +check("json-schema: a remote $ref is recorded as unsupported", + any(u.kind == "external-reference" for u in schema.unsupported_content)) +check("json-schema: the remote $ref is left unresolved", + not any("resolved type" in b.text and "example.invalid" in b.text + for b in schema.blocks)) +check("json-schema: the parse still succeeds without the remote target", + schema.status == DocumentParseStatus.PARSED) + +remote_api = b"""openapi: 3.0.0 +info: {title: R, version: '1'} +paths: + /x: + get: + responses: + '200': + content: + application/json: + schema: + $ref: 'https://example.invalid/schemas/x.json' +""" +api = parse_bytes(remote_api, logical_key="documents/r.yaml", filename="r.yaml") +check("openapi: a remote $ref is recorded as unsupported", + any(u.kind == "external-reference" for u in api.unsupported_content)) + +# YAML must never construct arbitrary Python objects. +evil_yaml = (b"openapi: 3.0.0\ninfo: {title: X, version: '1'}\n" + b"paths: !!python/object/apply:os.system ['echo pwned']\n") +yaml_doc = parse_bytes(evil_yaml, logical_key="documents/e.yaml", + filename="e.yaml") +check("openapi: a YAML python/object tag is refused (safe_load only)", + yaml_doc.status in (DocumentParseStatus.UNSUPPORTED, + DocumentParseStatus.FAILED)) + +# --------------------------------------------------------------------------- +# 8. Oversized inputs +# --------------------------------------------------------------------------- +big_pdf = parse_bytes((FIXTURES / "sample-design.pdf").read_bytes(), + logical_key="documents/b.pdf", filename="b.pdf", + limits=DocumentLimits(max_bytes=200)) +check("an oversized PDF is refused before parsing", + big_pdf.status == DocumentParseStatus.UNSUPPORTED) +check("the oversize refusal names DOCUMENT_MAX_BYTES", + any(w.detail.get("limit") == "DOCUMENT_MAX_BYTES" + for w in big_pdf.warnings)) + +huge_xlsx = parse_bytes((FIXTURES / "sample-tests.xlsx").read_bytes(), + logical_key="documents/h.xlsx", filename="h.xlsx", + limits=DocumentLimits(xlsx_max_cells=3)) +check("an oversized workbook becomes partial, per policy", + huge_xlsx.status == DocumentParseStatus.PARTIAL) +check("the workbook cap is named in the warning", + any(w.detail.get("limit") in ("XLSX_MAX_CELLS", "XLSX_MAX_ROWS_PER_SHEET") + for w in huge_xlsx.warnings)) +check("content read before the cap is KEPT, not discarded", + any(b.text.strip() for b in huge_xlsx.blocks)) + +blocky = parse_bytes(b"\n\n".join(b"paragraph %d" % i for i in range(50)), + logical_key="documents/many.txt", filename="many.txt", + limits=DocumentLimits(max_blocks=5)) +check("a block-count cap produces partial", + blocky.status == DocumentParseStatus.PARTIAL) +check("the block cap emits exactly ONE warning, not one per refused block", + sum(1 for w in blocky.warnings + if w.detail.get("limit") == "DOCUMENT_MAX_BLOCKS") == 1) +check("the block cap is actually enforced", len(blocky.blocks) <= 5) + +long_block = parse_bytes(b"# T\n\n" + b"x" * 5000, + logical_key="documents/l.md", filename="l.md", + limits=DocumentLimits(max_block_chars=200)) +check("an oversized block is truncated to the limit", + all(len(b.text) <= 200 for b in long_block.blocks)) +check("block truncation is REPORTED, never silent", + long_block.status == DocumentParseStatus.PARTIAL + and any(w.detail.get("limit") == "DOCUMENT_MAX_BLOCK_CHARS" + for w in long_block.warnings)) +check("a truncated block says so in its text", + any("truncated by OpenMind" in b.text for b in long_block.blocks)) + +# --------------------------------------------------------------------------- +# 9. Decoding never fails, and a lossy decode is disclosed +# --------------------------------------------------------------------------- +text, encoding, lossy = security.decode_text("héllo wörld".encode("utf-8")) +check("decode: UTF-8 round-trips", text == "héllo wörld" and not lossy) +text, encoding, lossy = security.decode_text(b"\xff\xfeh\x00i\x00") +check("decode: a UTF-16 BOM is honoured", text == "hi") +text, encoding, lossy = security.decode_text(b"caf\xe9 latte") +check("decode: undecodable UTF-8 falls back rather than raising", + text.startswith("caf") and text.endswith(" latte")) +check("decode: a single-byte fallback is NOT mis-decoded as UTF-16", + encoding in ("cp1252", "latin-1")) +mixed = parse_bytes(b"# T\n\ncaf\xe9 body\n", logical_key="documents/m.md", + filename="m.md") +check("decode: a document with a non-UTF-8 encoding still parses", + mixed.status in (DocumentParseStatus.PARSED, DocumentParseStatus.PARTIAL)) +check("decode: the encoding actually used is recorded", + bool(mixed.metadata.extra.get("encoding"))) + +# --------------------------------------------------------------------------- +# 10. A malformed document never escalates into a crash +# --------------------------------------------------------------------------- +for name, payload in (("truncated docx", (FIXTURES / "sample-requirements.docx") + .read_bytes()[:400]), + ("truncated pdf", (FIXTURES / "sample-design.pdf") + .read_bytes()[:120]), + ("truncated xlsx", (FIXTURES / "sample-tests.xlsx") + .read_bytes()[:300]), + ("empty file", b""), + ("random bytes", os.urandom(4096))): + result = parse_bytes(payload, logical_key="documents/x", filename="x") + check(f"a {name} returns a status instead of raising", + result.status in DocumentParseStatus.VALUES) + check(f"a {name} never claims to be fully parsed with fabricated content", + result.status != DocumentParseStatus.PARSED + or not [b for b in result.blocks if b.indexable]) + +# --------------------------------------------------------------------------- +bad = [d for d, ok in _results if not ok] +print(f"\n{len(_results) - len(bad)} passed, {len(bad)} failed") +sys.exit(1 if bad else 0) diff --git a/tests/verify_migrations.py b/tests/verify_migrations.py index 0629222..890a7c8 100644 --- a/tests/verify_migrations.py +++ b/tests/verify_migrations.py @@ -57,9 +57,10 @@ def apply(conn): result = migrations.migrate(conn) tables = _tables(conn) -check("empty db: runner reports the head version", result.version == 3) +check("empty db: runner reports the head version", result.version == 4) check("empty db: every migration is applied in order", - result.applied == ["0001_baseline", "0002_paths_sidecar", "0003_asset_model"]) + result.applied == ["0001_baseline", "0002_paths_sidecar", "0003_asset_model", + "0004_document_ingestion"]) check("empty db: nothing was reported as already applied", result.already_applied == []) check("empty db: not flagged as a legacy baseline", result.baselined_legacy is False) check("empty db: schema_migrations ledger exists", "schema_migrations" in tables) @@ -74,6 +75,19 @@ def apply(conn): "idx_revisions_blob", "idx_segments_revision", "idx_segments_symbol", "idx_segments_rev_type", "idx_evidence_revision", "idx_evidence_segment"): check(f"empty db: v0003 index '{i}' created", i in _idx) +# v0004 document-ingestion tables, columns and indexes +for t in ("document_parses", "document_index"): + check(f"empty db: v0004 table '{t}' created", t in tables) +for i in ("idx_document_parses_status", "idx_document_parses_parser", + "idx_document_index_ws", "idx_document_index_revision", + "idx_segments_blob"): + check(f"empty db: v0004 index '{i}' created", i in _idx) +check("empty db: v0004 added segments.content_blob_hash", + "content_blob_hash" in {r[1] for r in + conn.execute("PRAGMA table_info(segments)")}) +check("empty db: v0004 added jobs.payload_json", + "payload_json" in {r[1] for r in + conn.execute("PRAGMA table_info(jobs)")}) check("empty db: ask_history index created", conn.execute("SELECT 1 FROM sqlite_master WHERE type='index' AND " "name='idx_ask_scope'").fetchone() is not None) @@ -88,7 +102,7 @@ def apply(conn): check("repeat run: applies nothing", again.applied == []) check("repeat run: reports every migration as already applied", again.already_applied == ["0001_baseline", "0002_paths_sidecar", - "0003_asset_model"]) + "0003_asset_model", "0004_document_ingestion"]) check("repeat run: version is unchanged", again.version == result.version) # --------------------------------------------------------------------------- @@ -138,13 +152,20 @@ def apply(conn): legacy_result = migrations.migrate(legacy) check("legacy db: reported as a legacy baseline", legacy_result.baselined_legacy is True) -check("legacy db: brought to head version", legacy_result.version == 3) +check("legacy db: brought to head version", legacy_result.version == 4) check("legacy db: baseline recorded, not skipped", "0001_baseline" in legacy_result.applied) check("legacy db: v0003 applied on top of the baseline", "0003_asset_model" in legacy_result.applied) check("legacy db: v0003 asset tables created on the legacy database", {"assets", "asset_revisions", "segments", "evidence"} <= _tables(legacy)) +check("legacy db: v0004 applied on top of the baseline", + "0004_document_ingestion" in legacy_result.applied) +check("legacy db: v0004 document tables created on the legacy database", + {"document_parses", "document_index"} <= _tables(legacy)) +check("legacy db: v0004 added payload_json to the legacy jobs table", + "payload_json" in {r[1] for r in + legacy.execute("PRAGMA table_info(jobs)")}) check("legacy db: project row survived", legacy.execute("SELECT COUNT(*) FROM projects").fetchone()[0] == 1) check("legacy db: project meta survived intact",