From 3a5af3ec225c4b8d8a74a471b75d64519ae410d1 Mon Sep 17 00:00:00 2001 From: HelloThisWorld Date: Mon, 20 Jul 2026 23:32:41 +0800 Subject: [PATCH 1/9] Phase 3 (1/4): document parser SPI, registry, ten parsers, migration v0004 Adds the deterministic, model-free document parsing plane that later steps persist and index. - docs/v2/phase-3-document-ingestion.md: the required design document. - openmind/documents/: parser SPI (probe -> registry -> normalized ParsedDocument), a shared block builder that enforces every limit in one place, and parsers for Markdown, plain text/RST/AsciiDoc, HTML, CSV, DOCX, PDF, XLSX, OpenAPI, JSON Schema and SQL DDL. - Selection never uses the extension alone: OOXML formats must show package structure (word/document.xml, xl/workbook.xml) and PDF must show %PDF-. Exactly one parser is selected; a tie at equal priority fails loudly. - Honest statuses throughout: partial on a limit (with the exact warning), needs-ocr for an image-only PDF (no OCR is performed), encrypted, unsupported and failed - never an empty successful parse. - Security: ZIP member/size/ratio/traversal caps before any OOXML parse, defusedxml.defuse_stdlib() required for OOXML, formulas read as text and never evaluated, HTML scripts/styles/hidden elements dropped, remote $ref never fetched. - Migration v0004: segments.content_blob_hash, jobs.payload_json, document_parses and document_index. Additive and idempotent. - Fixtures: 11 invented NameCheck documents; the 5 binary ones regenerate byte-identically via scripts/build_document_fixtures.py. Parser imports stay lazy, so importing openmind.documents pulls in no document dependency and the dependency-free artifact export is unaffected. --- docs/v2/phase-3-document-ingestion.md | 675 ++++++++++++++++++ fixtures/documents/README.md | 58 ++ fixtures/documents/sample-cases.csv | 5 + fixtures/documents/sample-design.html | 39 + fixtures/documents/sample-design.pdf | Bin 0 -> 1359 bytes fixtures/documents/sample-encrypted.pdf | Bin 0 -> 1035 bytes fixtures/documents/sample-openapi.yaml | 64 ++ fixtures/documents/sample-requirements.docx | Bin 0 -> 37670 bytes fixtures/documents/sample-requirements.md | 47 ++ fixtures/documents/sample-scanned.pdf | Bin 0 -> 467 bytes fixtures/documents/sample-schema.json | 44 ++ fixtures/documents/sample-schema.sql | 33 + fixtures/documents/sample-tests.xlsx | Bin 0 -> 6161 bytes openmind/config.py | 64 ++ openmind/documents/__init__.py | 60 ++ openmind/documents/builder.py | 180 +++++ openmind/documents/csv_parser.py | 236 ++++++ openmind/documents/docx_parser.py | 330 +++++++++ openmind/documents/html_parser.py | 346 +++++++++ openmind/documents/json_schema_parser.py | 260 +++++++ openmind/documents/markdown_parser.py | 300 ++++++++ openmind/documents/models.py | 404 +++++++++++ openmind/documents/openapi_parser.py | 353 +++++++++ openmind/documents/pdf_parser.py | 248 +++++++ openmind/documents/probe.py | 155 ++++ openmind/documents/registry.py | 281 ++++++++ openmind/documents/security.py | 216 ++++++ openmind/documents/spreadsheet_parser.py | 350 +++++++++ openmind/documents/sql_parser.py | 346 +++++++++ openmind/documents/text_parser.py | 264 +++++++ openmind/domain/__init__.py | 15 +- openmind/domain/types.py | 249 ++++++- .../versions/v0004_document_ingestion.py | 123 ++++ scripts/build_document_fixtures.py | 344 +++++++++ 34 files changed, 6082 insertions(+), 7 deletions(-) create mode 100644 docs/v2/phase-3-document-ingestion.md create mode 100644 fixtures/documents/README.md create mode 100644 fixtures/documents/sample-cases.csv create mode 100644 fixtures/documents/sample-design.html create mode 100644 fixtures/documents/sample-design.pdf create mode 100644 fixtures/documents/sample-encrypted.pdf create mode 100644 fixtures/documents/sample-openapi.yaml create mode 100644 fixtures/documents/sample-requirements.docx create mode 100644 fixtures/documents/sample-requirements.md create mode 100644 fixtures/documents/sample-scanned.pdf create mode 100644 fixtures/documents/sample-schema.json create mode 100644 fixtures/documents/sample-schema.sql create mode 100644 fixtures/documents/sample-tests.xlsx create mode 100644 openmind/documents/__init__.py create mode 100644 openmind/documents/builder.py create mode 100644 openmind/documents/csv_parser.py create mode 100644 openmind/documents/docx_parser.py create mode 100644 openmind/documents/html_parser.py create mode 100644 openmind/documents/json_schema_parser.py create mode 100644 openmind/documents/markdown_parser.py create mode 100644 openmind/documents/models.py create mode 100644 openmind/documents/openapi_parser.py create mode 100644 openmind/documents/pdf_parser.py create mode 100644 openmind/documents/probe.py create mode 100644 openmind/documents/registry.py create mode 100644 openmind/documents/security.py create mode 100644 openmind/documents/spreadsheet_parser.py create mode 100644 openmind/documents/sql_parser.py create mode 100644 openmind/documents/text_parser.py create mode 100644 openmind/migrations/versions/v0004_document_ingestion.py create mode 100644 scripts/build_document_fixtures.py diff --git a/docs/v2/phase-3-document-ingestion.md b/docs/v2/phase-3-document-ingestion.md new file mode 100644 index 0000000..3e23165 --- /dev/null +++ b/docs/v2/phase-3-document-ingestion.md @@ -0,0 +1,675 @@ +# OpenMind v2 — Phase 3: Enterprise Document Ingestion and Appendable Knowledge + +Status: implemented on branch `feat/v2-phase3-document-ingestion`. +Runtime version introduced by this phase: **`1.3.0-dev`**. +Artifact export contract: **unchanged** — `.openmind` schema stays `1.1.0`. + +Phase 3 makes enterprise documents first-class OpenMind Assets. It adds a +deterministic, model-free document-ingestion plane on top of the Phase 2 +Asset/Revision/Segment/Evidence foundation, and it deliberately implements +**none** of the semantic layer above it — no Requirement extraction, no Business +Rules, no Design Decisions, no Claim/Relation tables, no Knowledge-Graph edges, +no OCR, no cloud model calls. See [§16 Deferred work](#16-deferred-phase-4-work). + +--- + +## 1. Baseline: what already worked + +Phase 3 started from a clean working tree at `3a5f6e9` (branch `main`), with the +whole core acceptance suite green. Recorded before any change was made +(`python scripts/run_acceptance.py --json`): + +``` +ok=true — 28 passed, 0 failed, 0 skipped +``` + +| Script | Result | Script | Result | +| --- | --- | --- | --- | +| verify_migrations | 64 passed | verify_facets | 22/22 | +| verify_content_store | 23 passed | verify_guide | 19/19 | +| verify_structure | 14 passed | verify | 17/17 | +| verify_glossary | 20 passed | verify_fixes | 7/7 | +| verify_router | 23 passed | verify_fixes2 | 7/7 | +| verify_diagrams | 7 passed | verify_resources | 15 passed | +| verify_grounding | 8 passed | verify_ask | 15/15 | +| verify_artifacts | 31 passed | verify_ask2 | 29/29 | +| verify_runtime | 31 passed | verify_async_delete | 6/6 | +| verify_services | 97 passed | verify_delete_race | 26/26 | +| verify_cli | 112 passed | verify_source_link | 25 passed | +| verify_adapters | 88 passed | verify_modelserver | 10/10 | +| verify_asset_model | 75 passed | verify_templates | 21/21 | +| verify_asset_cli | 34 passed | verify_asset_adapters | 31 passed | + +Schema head before this phase: **version 3** (`v0001_baseline`, +`v0002_paths_sidecar`, `v0003_asset_model`). +MCP tool set before this phase: **13** — the nine core tools +(`search`, `route`, `dispatch`, `get_glossary`, `find_similar_cases`, +`save_case`, `get_doc`, `propose_fix`, `apply_fix`) plus the four Phase 2 Asset +tools (`list_assets`, `get_asset`, `get_asset_revisions`, `get_evidence`). + +--- + +## 2. Phase 2 Asset model (what this phase builds on) + +``` +Workspace (= the existing project row; id p_*) +└── Asset one logical engineering object, keyed (workspace_id, logical_key) + └── AssetRevision an immutable observation of that Asset's bytes + ├── Segment a stable structural unit inside one revision + └── Evidence a source-locatable citation for a segment +``` + +Facts Phase 3 relies on and does not change: + +* `assets.logical_key` is workspace-relative and portable; no absolute machine + path is ever stored in the database (the source root lives in the + machine-local sidecar, `openmind/machine.py`). +* `asset_revisions.content_blob_hash` is the SHA-256 of the revision's **exact + bytes**, stored in the immutable content store + (`data//objects//`). +* `db.commit_revision` is the single transactional writer: an Asset's + `current_revision_id` is repointed **last**, so it can never name a revision + whose segments and evidence are not yet committed. +* Unchanged content creates **no** new revision (idempotent, revert-safe), and a + removed-then-reappearing Asset is reactivated rather than duplicated. +* Chroma is a *retrieval projection*, never the source of truth. + +Phase 3 adds a parallel, additive plane. It does not modify `v0003`, does not +change `commit_revision`'s existing behaviour, and does not move code Segments. + +--- + +## 3. Target document-ingestion architecture + +``` +Local files under registered roots Manually attached document + │ │ + └──────────────┬────────────────────────┘ + ▼ + Document Intake Service (openmind/documents/intake.py) + │ read bytes → SHA-256 → immutable staged blob + ▼ + Parser Registry (openmind/documents/registry.py) + │ probe → exactly one parser (or a typed failure) + ▼ + Normalized ParsedDocument (openmind/documents/models.py) + metadata · blocks · warnings · coverage + │ + ▼ + Asset / Revision / Segment / Evidence + document_parses + │ + ┌──────────────┴───────────────┐ + ▼ ▼ + documents_ collection deterministic candidate association + │ │ + └──────────────┬───────────────┘ + ▼ + CLI · REST · MCP · Claude Code +``` + +Raw bytes and normalized content stay on the machine. No project document +content leaves the machine in this phase; no parser dependency performs network +I/O or telemetry. + +--- + +## 4. Parser SPI + +`openmind/documents/` holds the whole plane: + +``` +openmind/documents/ +├── __init__.py lazy re-exports; importing it pulls in NO parser dependency +├── models.py ParsedDocument, DocumentBlock, warnings, probe, context +├── security.py limits, ZIP package safety, safe XML, bounded decoding +├── registry.py register / list_parsers / select / parse (lazy imports) +├── probe.py magic + package-structure detection (never extension alone) +├── text_parser.py plain text, RST, AsciiDoc +├── markdown_parser.py Markdown / CommonMark subset +├── html_parser.py HTML (stdlib HTMLParser, scripts and styles dropped) +├── csv_parser.py CSV / TSV (stdlib csv, bounded sniffing) +├── docx_parser.py DOCX (python-docx) +├── pdf_parser.py PDF (pypdf) +├── spreadsheet_parser.py XLSX (openpyxl) +├── openapi_parser.py OpenAPI 2/3 (JSON + YAML) +├── json_schema_parser.py JSON Schema +├── sql_parser.py SQL DDL (sqlglot, honest text fallback) +├── pipeline.py parse → blobs → segments → transactional commit → vectors +└── candidates.py deterministic candidate association +``` + +### 4.1 Contract + +```python +class DocumentParser(Protocol): + name: str + version: str + def supports(self, probe: DocumentProbe) -> bool: ... + def parse(self, content: bytes, context: DocumentParseContext) -> ParsedDocument: ... +``` + +`DocumentProbe` carries `filename`, `extension`, `declared_media_type`, +`detected_media_type`, `magic` (signature facts: `zip`, `pdf`, `ole2`, `utf8`, +`bom`, `zip_members` for OOXML package structure), `size`, and a bounded `head` +sample of the bytes. **A parser is never selected from the extension alone.** +For ZIP-based OOXML formats the probe verifies package structure +(`word/document.xml` for DOCX, `xl/workbook.xml` for XLSX) before a parser may +claim the file. + +`DocumentParseContext` carries the portable `logical_key`, the original +`filename`, the workspace id, the effective `DocumentLimits`, and free-form +`parser_options`. + +### 4.2 Registry rules + +1. **Deterministic order** — parsers are registered with an explicit integer + priority; ties break on parser name. `list_parsers()` returns that order. +2. **Exactly one parser is selected.** `select(probe)` returns the single + highest-priority parser whose `supports()` is true. +3. **Ambiguity fails clearly.** Two parsers at the *same* priority both claiming + a probe raise `AmbiguousParser` rather than silently picking one. +4. **Missing optional dependency** → `ParsedDocument(status="unsupported")` with + `reason="dependency_unavailable"` and the missing distribution named. Only + that format fails. +5. **Unsupported format** → `status="unsupported"`, never an empty successful + parse. +6. **Parser imports are lazy** — `registry.select()` imports a parser module + only when its probe matches, and the third-party library only inside + `parse()`. Importing `openmind.documents` imports no parser dependency. +7. **Artifact export stays dependency-free**: `openmind export` never imports + `openmind.documents`. + +--- + +## 5. Normalized document model + +``` +ParsedDocument +├── parser_name, parser_version, schema_version ("1.0") +├── status parsed | partial | needs-ocr | encrypted | unsupported | failed +├── title, media_type +├── metadata DocumentMetadata (author/created/modified/producer/version_label/extra) +├── blocks [DocumentBlock] +├── warnings [DocumentParseWarning] (code, message, locator?) +├── unsupported_content [UnsupportedContent] (kind, count, detail) +└── coverage {"blocks": n, "indexable": n, "truncated": bool, ...} +``` + +``` +DocumentBlock +├── block_key stable, deterministic, unique within the document +├── block_type document | section | heading | paragraph | list-item | code-block +│ | table | table-row | sheet | cell-range | page | api-operation +│ | schema-definition | sql-object +├── ordinal dense, 0-based, document order +├── parent_key the containing block's key ("" for the root) +├── heading_path [str] ancestor headings, outermost first +├── text the represented text +├── content_mode verbatim | derived +├── locator a portable document locator (see §6) +├── metadata {} format-specific facts +└── indexable whether this block is embedded into the document collection +``` + +**Content mode.** `verbatim` is used only when `text` is an exact textual +representation of the source. A synthesized table rendering, a spreadsheet row +serialized as `header=value` pairs, an OpenAPI operation summary and the +document root block are `derived`. Markdown/text/HTML paragraphs, DOCX +paragraph runs, PDF page text and code blocks are `verbatim`. + +**Indexable.** Structural containers (the `document` root, an empty `section`, +a `table` wrapper whose rows carry the content) are stored as Segments but not +embedded. Only content-bearing leaves are indexed, so retrieval is not diluted +by empty scaffolding. Every stored block still becomes a Segment with Evidence. + +--- + +## 6. Document source locators + +The Phase 2 `source-range` locator remains valid for code and is untouched. +Document locators are additive and always carry a **portable logical document +key**, never an absolute path. Page and line numbers presented to users are +1-based. + +| Format | `kind` | Fields | +| --- | --- | --- | +| Markdown / text / RST / AsciiDoc | `text-range` | `document`, `startLine`, `endLine`, `headingPath` | +| HTML | `html-element` | `document`, `element`, `elementIndex`, `domPath`, `headingPath` | +| DOCX paragraph | `docx-paragraph` | `document`, `paragraphIndex`, `headingPath` | +| DOCX table row | `docx-table-row` | `document`, `tableIndex`, `rowIndex`, `cellRange` | +| PDF | `pdf-block` | `document`, `page`, `blockIndex` | +| XLSX | `spreadsheet-range` | `document`, `sheet`, `range` | +| CSV | `spreadsheet-range` | `document`, `sheet` (`""`), `range`, `rowIndex` | +| OpenAPI / JSON Schema | `json-pointer` | `document`, `pointer` | +| SQL DDL | `text-range` | `document`, `startLine`, `endLine`, `symbol` | + +--- + +## 7. Segment-content snapshots + +Phase 2 Evidence for **code** is reconstructed by slicing `[startLine, endLine]` +out of the revision blob. That is safe because the mapping from bytes to lines +is fixed forever. + +For a **document** it is not. Re-deriving a DOCX paragraph or a PDF page block +requires re-running a parser, and a future parser version may legitimately +produce different block boundaries. Reconstructing historical Evidence that way +would silently rewrite history. + +So Phase 3 adds an optional **segment content blob**: + +```sql +ALTER TABLE segments ADD COLUMN content_blob_hash TEXT NOT NULL DEFAULT ''; +``` + +For a document segment the pipeline + +1. stores the block's exact represented text (UTF-8) in the content store, +2. records that blob's SHA-256 in `segments.content_blob_hash`, +3. sets `segments.content_hash` to the SHA-256 of that same exact text, +4. sets the Evidence `content_hash` to the same value. + +Historical document Evidence is then recovered **from the block blob**, with no +parser rerun. Existing code segments keep `content_blob_hash = ''` and continue +to resolve through the line-range path; Phase 2 rows are **not** backfilled. + +--- + +## 8. Document parse records + +```sql +CREATE TABLE document_parses ( + revision_id TEXT PRIMARY KEY, + parser_name TEXT NOT NULL, + parser_version TEXT NOT NULL, + schema_version TEXT NOT NULL, + status TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + media_type TEXT NOT NULL DEFAULT '', + metadata_json TEXT NOT NULL DEFAULT '{}', + warnings_json TEXT NOT NULL DEFAULT '[]', + unsupported_json TEXT NOT NULL DEFAULT '[]', + coverage_json TEXT NOT NULL DEFAULT '{}', + structure_hash TEXT NOT NULL, + created_at TEXT NOT NULL, + FOREIGN KEY(revision_id) REFERENCES asset_revisions(id) ON DELETE CASCADE +) +``` + +It is a **derived parse projection over an immutable Revision**. In Phase 3 a +Revision gets exactly one parse result; the parser name and version are +recorded; the output is deterministic (`structure_hash` is the SHA-256 over the +ordered `(block_key, block_type, ordinal, parent_key, content_hash)` tuples, so +a re-parse that changes structure is detectable). Old Revisions are never +automatically re-parsed, and a historical parse is never silently replaced by a +different parser version. A future analysis-generation model may allow that. + +**The presence of a `document_parses` row for an Asset's current Revision is the +definition of "this Asset is a document."** That is a recorded fact, not an +inference. + +--- + +## 9. Document vector projection + +Documents get their **own collection per workspace** so the existing `search` +contract cannot change: + +``` +code_ unchanged — code/config chunks (existing `search`) +cases_ unchanged +documents_ NEW — document block chunks +``` + +```sql +CREATE TABLE document_index ( + workspace_id TEXT NOT NULL, + asset_id TEXT NOT NULL, + revision_id TEXT NOT NULL, + chunk_ids_json TEXT NOT NULL DEFAULT '[]', + updated_at TEXT NOT NULL, + PRIMARY KEY(workspace_id, asset_id), + FOREIGN KEY(asset_id) REFERENCES assets(id) ON DELETE CASCADE +) +``` + +Each indexed chunk carries: `workspace_id`, `asset_id`, `revision_id`, +`segment_id`, `evidence_id`, `logical_key`, `title`, `asset_type`, `block_type`, +`heading_path`, `parser_name`, `page`, `sheet`, `json_pointer`, `content_hash`. +Only relevant fields are populated. + +**Active-revision behaviour.** Document search returns the current active +Revision. When a new Revision becomes current the pipeline removes the previous +active chunks, adds the new ones and updates `document_index`; every historical +Segment and Evidence row is preserved. Historical search may be added later. + +**Lifecycle.** `vectorstore._pid_of` recognizes the `documents_` prefix, so: +workspace terminate drops the document collection, workspace delete drops it, +and the startup orphan sweep classifies it correctly (a *deleting* workspace is +still "known" and therefore not an orphan). Drains stay batched, interruptible +and protected by the existing in-flight guard — the delete-race fixes are not +touched. + +Chunk ids are stable and content-derived +(`d_` + `sha1(asset_id|revision_id|block_key)[:18]`), so a retried upsert after +a mid-commit failure writes the same ids and is idempotent. + +--- + +## 10. Format support + +| Format | Parser | Support | Not supported in Phase 3 | Security limits | +| --- | --- | --- | --- | --- | +| Markdown | `markdown` | headings + hierarchy, paragraphs, lists, fenced code, block quotes, pipe tables, line locators | reference/footnote resolution, inline HTML rendering, MDX | `DOCUMENT_MAX_BYTES`, `DOCUMENT_MAX_BLOCKS`, `DOCUMENT_MAX_BLOCK_CHARS` | +| Plain text / RST / AsciiDoc | `text` | line ranges, blank-line paragraphs, underline/`=`/`#` headings, list items, indented code blocks | RST/AsciiDoc directives, includes, substitutions (reported as unsupported content) | as above | +| HTML | `html` | ``, headings, paragraphs, list items, tables, `pre`/`code`; scripts, styles, comments and `hidden` content dropped | JavaScript execution, external resource fetch, CSS-computed visibility | as above; no network | +| DOCX | `docx` | core properties, paragraphs, heading styles + hierarchy, list items, tables/rows/cells, code-like styles, section order | macros, embedded images (recorded as unsupported content), tracked changes, headers/footers | ZIP package caps, safe XML, `DOCX_MAX_PARAGRAPHS`, `DOCX_MAX_TABLES` | +| PDF (text) | `pdf` | metadata, per-page text, block ordering where available, 1-based page locators | OCR, table recovery, form fields, annotations | `PDF_MAX_PAGES`, `DOCUMENT_MAX_BYTES` | +| XLSX | `xlsx` | workbook props, visible sheets, bounded used ranges, row/cell values, formulas **as text**, merged-cell metadata, header detection | formula evaluation, external links, macros (`.xlsm` unsupported), charts | ZIP package caps, `XLSX_MAX_SHEETS`, `XLSX_MAX_ROWS_PER_SHEET`, `XLSX_MAX_CELLS` | +| CSV / TSV | `csv` | encoding fallback, bounded dialect sniffing, header row, row locators | nested/multi-table files, type inference | `CSV_MAX_ROWS`, `DOCUMENT_MAX_BLOCK_CHARS` | +| OpenAPI (JSON/YAML) | `openapi` | `info.title`/`info.version`, paths + operations, parameters, request bodies, responses, component schemas, JSON-Pointer locators | remote `$ref` resolution, business-meaning inference, validation | safe YAML (`yaml.safe_load`), `DOCUMENT_MAX_BLOCKS` | +| JSON Schema | `json-schema` | root, `definitions`/`$defs`, properties, constraints, internal `$ref` | remote `$ref` fetch, schema validation | as above; **no network** | +| SQL DDL | `sql` | tables, views, columns, indexes, constraints, sequences, line ranges; honest bounded-text fallback on unsupported dialect syntax | dialect-perfect parsing, execution | `DOCUMENT_MAX_BLOCKS` | + +**Truncation is never silent.** When a limit is reached the parser keeps the +already-extracted content, sets `status="partial"`, appends a warning naming the +exact limit and the observed value, and reports `coverage`. + +**Honest PDF statuses.** An image-only page produces no fabricated text. A PDF +with no meaningful extractable text becomes `needs-ocr`; an encrypted PDF that +cannot be opened with the empty password becomes `encrypted`. **No OCR is +performed in Phase 3** and no result ever claims OCR was performed. + +--- + +## 11. Security and resource controls + +Document parsing handles untrusted files, so the controls are explicit and +tested (`tests/verify_document_security.py`). + +**ZIP package safety** (`security.inspect_zip`) — for DOCX/XLSX: +member-count cap, total-uncompressed-size cap, per-member size cap, path +traversal rejection (`..`, absolute, drive-letter, backslash members), +compression-ratio (zip-bomb) detection. Members are read **in memory from the +already-validated archive**; nothing is ever extracted to a filesystem path, and +no embedded content is executed. + +**XML safety** — `defusedxml` is installed as a dependency and +`defusedxml.defuse_stdlib()` is applied before any OOXML parse, so external +entities, external DTDs and entity-expansion bombs are refused by the underlying +stdlib parsers that `python-docx`/`openpyxl` use. When `defusedxml` is absent the +DOCX/XLSX parsers report `dependency_unavailable` rather than parsing unsafely. + +**Configurable limits** (`openmind/config.py`, each overridable by an +`OPENMIND_*` environment variable): + +``` +DOCUMENT_MAX_BYTES 25_000_000 +PDF_MAX_PAGES 2_000 +DOCX_MAX_PARAGRAPHS 50_000 +DOCX_MAX_TABLES 2_000 +XLSX_MAX_SHEETS 100 +XLSX_MAX_ROWS_PER_SHEET 20_000 +XLSX_MAX_CELLS 500_000 +CSV_MAX_ROWS 50_000 +DOCUMENT_MAX_BLOCKS 20_000 +DOCUMENT_MAX_BLOCK_CHARS 20_000 +ZIP_MAX_MEMBERS 2_000 +ZIP_MAX_TOTAL_BYTES 400_000_000 +ZIP_MAX_MEMBER_BYTES 100_000_000 +ZIP_MAX_RATIO 200 +``` + +**Parsing isolation.** `registry.parse()` catches every parser exception and +returns `status="failed"` with the exception type and message. A parser failure +fails that document's job step, never the worker process, and never leaves a +partial current Revision. + +--- + +## 12. Append-document workflow and identity rules + +`DocumentService` (`runtime.documents`, `ServiceContainer.documents`) exposes +`plan_import`, `add_document`, `get_document`, `list_documents`, `get_outline`, +`search`, `search_knowledge`, `find_related_candidates`. + +``` +Read local bytes → SHA-256 → immutable staged blob (content store) + → persist a safe import payload (jobs.payload_json — NO absolute path) + → enqueue a `document_ingest` job + → parse from the staged blob + → Asset / Revision / Segments / Evidence + document_parses + → document vector projection + → deterministic candidate association + → import report +``` + +The job payload may contain only `staged_blob_hash`, `original_filename`, +`requested_asset_id`, `requested_logical_key`, `import_mode`, `version_label`, +`parser_options`. **The absolute origin path never enters the portable +database.** + +### 12.1 Identity + +| Origin | Logical key | +| --- | --- | +| Under a registered source root | `workspace_id` + workspace-relative path (unchanged Phase 2 rule) | +| Manually attached | `documents/<normalized-filename>` | +| `--logical-key K` | `K` (normalized) | +| `--new-asset` | `documents/<stem>--<content-hash-prefix>.<ext>` — readable and deterministic, never opaque-random | + +Two different documents are **never** silently merged because their filenames +match. + +### 12.2 Import decisions + +| Condition | `status` | Effect | +| --- | --- | --- | +| Content hash already exists as a document Revision in the workspace | `duplicate` | Returns the existing Asset + Revision. **No job, no Revision, no vector duplicate.** | +| `--asset A` supplied, content differs | `revision` | Next Revision of `A` (validated: belongs to the workspace, is document-compatible) | +| `--asset A` supplied, content unchanged | `duplicate` | As above | +| `--logical-key K` | `new_asset` / `revision` | Find or create that Asset | +| Default key exists with **different** content and no explicit choice | `possible_revision` | **Nothing is written.** Returns the candidate Asset, its current Revision, both content hashes and the exact retry commands. | +| `--new-asset` | `new_asset` | Distinct deterministic key | +| Nothing matches | `new_asset` | Create | + +### 12.3 Version label + +Only explicit deterministic metadata is used: OpenAPI `info.version`, the DOCX +core `revision` property, a clearly-defined PDF metadata version field, or a +caller-supplied `--version-label` (which always wins). Versions are **never** +inferred from prose or filename patterns. `revision.status` stays `unknown` — +approval authority is not inferred. + +### 12.4 Workspace discovery vs. code discovery + +`config` now separates the policies: + +``` +CODE_INDEX_EXTENSIONS = INDEX_EXTENSIONS (unchanged: .java .ts .yaml .sql .html …) +TEXT_DOCUMENT_EXTENSIONS = .md .markdown .rst .adoc .asciidoc .txt +BINARY_DOCUMENT_EXTENSIONS = .docx .pdf .xlsx +STRUCTURED_DOCUMENT_EXTENSIONS = .csv .tsv +DOCUMENT_DISCOVERY_EXTENSIONS = (the three above) − CODE_INDEX_EXTENSIONS +``` + +The subtraction is load-bearing: a file already owned by the **code** pipeline +(`.html`, `.yaml`, `.json`, `.sql`, …) is never also discovered as a workspace +document, so nothing is ingested twice. Those formats are still fully parseable +**on demand** through `document add`, which is how an OpenAPI YAML or a schema +SQL file becomes a document Asset. And no document extension is added to +`INDEX_EXTENSIONS`, so a `.pdf` can never reach `walker.read_text`. + +A **full** ingest discovers code assets, discovers document assets, runs both +pipelines, and marks removed assets in both. A **filtered** code ingest never +prunes document Assets, and a filtered document ingest never rebuilds or prunes +code Assets. + +**Removal.** A workspace document that disappears has its active document chunks +removed and its Asset marked `removed`; every Revision, Segment, Evidence row +and content blob is preserved. A **manually attached** snapshot never becomes +`removed` because its external origin path vanished — the snapshot is the +canonical source (`source_kind="attachment"`). + +--- + +## 13. Evidence retrieval + +`AssetService.get_evidence` keeps its exact Phase 2 behaviour for code Evidence +(source-range slice out of the revision blob + live-file comparison). Document +Evidence is resolved through an additive locator-resolver path: + +1. If the Segment has a `content_blob_hash`, the exact block text is read from + that blob and verified against the Evidence `content_hash` → `snapshot: + available` (or `corrupt` on mismatch). **No parser is rerun.** +2. `current_source.status` is: + * `not-applicable` for an attachment (its origin path is deliberately not + tracked, so `missing` would be a false claim); + * `matches` / `changed` / `missing` for a workspace document file, decided by + comparing the **whole current document's** content hash against the + Revision's — block-level re-resolution is not deterministic across parser + versions, so document-level comparison is the honest answer. + +The response adds `parser`, `revision`, `snapshot`, `current_source` and +`truncated`; bounded output truncates honestly. + +--- + +## 14. Retrieval + +### 14.1 Document search (`openmind/document_rag.py`) + +`vector retrieval + lexical retrieval + exact identifier matching`, fused with +**RRF**, bounded. An exact Requirement ID, API path, error code or identifier is +matched on token boundaries and is **promoted above** any merely +embedding-similar result — the same guarantee the code RAG already gives. + +Each hit carries `asset_id`, `revision_id`, `segment_id`, `evidence_id`, +`title`, `logical_key`, `block_type`, `heading_path`, `locator`, `excerpt`, +`score`, `retrieval_sources`. Filters: `asset_type`, `parser`, `block_type`, +`logical_key`. Removed Assets are excluded by default. + +### 14.2 Combined knowledge search + +`search_knowledge` returns code and documents **separately**: + +```json +{"query": "...", + "code": {"hits": []}, + "documents": {"hits": []}, + "grounding": {"codeCount": 0, "documentCount": 0}} +``` + +It never claims a document hit *implements* or *refines* a code hit. This is +retrieval, not relationship inference. The existing MCP `search` tool stays +code-oriented and byte-for-byte unchanged. + +### 14.3 Deterministic candidate association + +`find_related_candidates` compares an imported document against existing +knowledge using deterministic signals first: + +| # | Signal | Candidate type | Confidence | +| --- | --- | --- | --- | +| 1 | exact workspace file path | `mentions-file` | high | +| 2 | exact code symbol (Segment `symbol`) | `mentions-symbol` | high | +| 3 | Requirement-like identifier (`REQ-NC-017`) | `mentions-document` / `similar-content` | high | +| 4 | change-request / ticket identifier (`ABC-1234`) | `mentions-document` | high | +| 5 | API path + HTTP method | `mentions-api` | medium | +| 6 | error code | `mentions-configuration` | medium | +| 7 | configuration key (dotted) | `mentions-configuration` | medium | +| 8 | database object | `mentions-database-object` | medium | +| 9 | message / topic name | `mentions-configuration` | medium | +| 10 | exact glossary term | `mentions-document` | high | +| 11 | semantic retrieval fallback | `similar-content` / `possibly-related` | low | + +Every candidate carries `candidate_type`, `confidence`, `reason`, +`document_evidence`, `target`, `target_evidence`, `retrieval_method` and +`status: "candidate"`. `implements`, `refines`, `verifies` and `contradicts` are +**never** returned — they require the semantic verification deferred to Phase 4. +**Candidates are computed on demand and are never persisted as canonical +Relations.** + +--- + +## 15. Compatibility boundaries + +| Surface | Guarantee | +| --- | --- | +| Runtime | CLI, MCP, FastAPI and tests keep using the same `OpenMindRuntime`; `documents` is an additive service property. | +| REST | No route removed or renamed. The public API keeps saying `/projects`, never `/workspaces`. All Phase 3 routes are additive. `/ocr` stays separate — a PDF import is **never** silently routed through OCR. | +| MCP | The nine core tools and the four Phase 2 Asset tools are unchanged. Six read-only document tools are added; **no document-write MCP tool exists in Phase 3** (Claude Code drives imports through the CLI). MCP startup still does not start the worker. | +| Artifact | `.openmind schemaVersion` stays `1.1.0`; the document model is **not** exported through Bundle 1.x. | +| Skill bridge | JSON-lines protocol unchanged, still independent of the application database. | +| Job engine | The single worker is not replaced. `document_ingest` is an additional job type; `jobs.payload_json` is an additive nullable column. | +| Databases | A Phase 1 or Phase 2 database migrates to head with projects, paths, jobs, Asset history, content blobs, file index, vector collections, maps, cases, Ask history and template metadata intact — `v0004` only `CREATE`s tables/indexes and `ADD COLUMN`s with defaults. | + +### 15.1 Schema (migration `v0004_document_ingestion`) + +* `ALTER TABLE segments ADD COLUMN content_blob_hash TEXT NOT NULL DEFAULT ''` +* `ALTER TABLE jobs ADD COLUMN payload_json TEXT NOT NULL DEFAULT '{}'` +* `CREATE TABLE document_parses (…)` +* `CREATE TABLE document_index (…)` +* indexes: `idx_document_parses_status`, `idx_document_index_ws`, + `idx_segments_blob` + +The migration is written so it is safe to re-run against a database that already +has the column (SQLite has no `ADD COLUMN IF NOT EXISTS`), via an `upgrade(conn)` +function that inspects `PRAGMA table_info` first. + +### 15.2 Dependency policy + +| Package | License | Why | Used by | +| --- | --- | --- | --- | +| `python-docx` | MIT | DOCX paragraphs/tables/styles | `docx_parser` | +| `pypdf` | BSD-3-Clause | pure-Python PDF text extraction | `pdf_parser` | +| `openpyxl` | MIT | XLSX cells, formulas-as-text, merges | `spreadsheet_parser` | +| `defusedxml` | PSF-2.0 | hardens the stdlib XML parsers OOXML uses | `security` | + +All four are permissive and compatible with this MIT project. **No AGPL parser +dependency is introduced** — in particular PyMuPDF is deliberately *not* used +for PDF text extraction. None of the four performs telemetry or network calls. +All are imported lazily, a missing one fails only its format, and the +dependency-free `.openmind` artifact export CI job stays green. + +--- + +## 16. Testing strategy + +New acceptance scripts, all registered in `scripts/run_acceptance.py` (an +unregistered `tests/verify_*.py` already fails the runner, so a missing core +document test fails the manifest): + +| Script | Covers | +| --- | --- | +| `verify_document_registry` | probe facts, deterministic order, single selection, ambiguity failure, `dependency_unavailable`, `unsupported`, lazy imports | +| `verify_document_parsers` | every mandatory per-format case in the task spec, plus byte-for-byte determinism on a repeated parse | +| `verify_document_security` | ZIP traversal, zip bomb, member caps, XML entities disabled, oversized PDF, oversized workbook, formulas not executed, HTML scripts ignored, no remote `$ref` fetch | +| `verify_document_ingest` | the 13 mandatory import cases + the mandatory Evidence cases | +| `verify_document_search` | exact-ID retrieval, exact-outranks-semantic, evidence ids, combined search shape, candidate association, low-confidence labelling, never-confirmed | +| `verify_document_cli` | `document add/list/show/outline/search/related`, `knowledge search`, JSON contract, exit codes, bounds, `--dry-run` | +| `verify_document_adapters` | additive REST routes, cross-workspace 404, the 13 pre-existing MCP tools still present, the 6 new document tools additive | + +Fixtures live in `fixtures/documents/` and are **invented, neutral content** +about a fictional "NameCheck" service. `sample-requirements.docx`, +`sample-design.pdf` and `sample-tests.xlsx` are generated deterministically by +`scripts/build_document_fixtures.py` (documented in the fixture README) and are +each a few kilobytes. + +--- + +## 17. Deferred (Phase 4+) work + +Explicitly **not** implemented here, and never described as implemented: + +* Requirement, Business Rule, Design Decision and Acceptance Criterion + extraction; +* semantic document classification and document-authority inference; +* canonical Claim and Relation tables, Knowledge-Graph edges, + Requirement-to-Code traceability, conflict detection, induced Project Lens; +* OCR execution (image-only PDFs are *detected* and marked `needs-ocr` only); +* COBOL / JCL / PPTX / email-archive parsing; Jira and Confluence connectors; +* branch or PR overlays, webhooks, Bundle 2.0, Titan Mind, Neo4j; +* cloud OpenAI / Anthropic / Bedrock / Azure / Vertex calls; +* historical (non-current-revision) document search; +* a worker-pool rewrite, new Agent Skills, and any new UI. diff --git a/fixtures/documents/README.md b/fixtures/documents/README.md new file mode 100644 index 0000000..313d055 --- /dev/null +++ b/fixtures/documents/README.md @@ -0,0 +1,58 @@ +# Document parser fixtures + +Small, neutral, **invented** documents for the Phase 3 document-ingestion tests. + +Everything here describes a fictional "NameCheck" screening service. No content +is copied from any real, proprietary or employer document, and no identifier, +endpoint, error code or requirement id in these files refers to a real system. + +## Text fixtures (checked in as source) + +These are ordinary text and are edited directly — a readable diff is worth more +than uniformity with the generated ones. + +| File | Exercises | +| --- | --- | +| `sample-requirements.md` | heading hierarchy, paragraphs, list items, a fenced code block, a pipe table, a block quote, exact line locators | +| `sample-design.html` | `<title>`, headings, paragraphs, lists, a table, `pre`/`code`; a `<script>`, a `<style>` and two hidden paragraphs that must **not** be indexed | +| `sample-cases.csv` | header detection, row locators, dialect sniffing | +| `sample-openapi.yaml` | `info.title`/`info.version`, path operations, a path-level parameter, request body, responses, component schemas, JSON-Pointer locators | +| `sample-schema.json` | root schema, `$defs`, properties, constraints, an internal `#/$defs` `$ref` that IS resolved, and a remote `$ref` that must **never** be fetched | +| `sample-schema.sql` | tables, columns, constraints, an index, a view, plus a PL/SQL package body that `sqlglot` cannot structure and that must fall back to bounded verbatim text with an exact line range | + +## Binary fixtures (generated) + +Run: + +```bash +python scripts/build_document_fixtures.py +``` + +| File | Built with | Exercises | +| --- | --- | --- | +| `sample-requirements.docx` | `python-docx` | core properties (including `revision` → version label), heading styles and hierarchy, list styles, a table, and one embedded image recorded as unsupported content | +| `sample-tests.xlsx` | `openpyxl` | workbook properties, two visible sheets and one hidden one, a merged banner cell, bounded used ranges, and two formulas that must be stored **as formulas** and never evaluated | +| `sample-design.pdf` | hand-written PDF bytes | metadata, two pages of extractable text, 1-based page locators | +| `sample-scanned.pdf` | hand-written PDF bytes | a page with a drawn rectangle and **no text operators** → must be reported `needs-ocr`, with no fabricated text | +| `sample-encrypted.pdf` | hand-written PDF bytes | an `/Encrypt` trailer that does not open with the empty password → must be reported `encrypted` | + +### Why the PDFs are hand-written + +Writing the PDF bytes directly keeps a PDF-*writing* dependency out of the +repository (OpenMind only ever *reads* PDFs), keeps each file well under 2 KB, +and puts the bytes fully under our control — which is what makes "this page has +no extractable text" and "this file is encrypted" reliable test inputs instead +of a library's changing idea of them. + +### Why regeneration is byte-identical + +An unchanged document must produce **no new Revision**. A fixture whose bytes +drifted between runs would change its content hash and make that guarantee +untestable. So every timestamp is pinned to a fixed epoch, and the OOXML +packages are rewritten with fixed member order, fixed member timestamps and +fixed compression (`normalize_zip` in the generator) — `openpyxl` in particular +overwrites `dcterms:modified` with the wall clock during `save()`, so that field +is rewritten afterwards. + +Regenerating and committing should therefore produce an empty diff. If it does +not, the generator changed and the change is real. diff --git a/fixtures/documents/sample-cases.csv b/fixtures/documents/sample-cases.csv new file mode 100644 index 0000000..1112b76 --- /dev/null +++ b/fixtures/documents/sample-cases.csv @@ -0,0 +1,5 @@ +Case ID,Requirement,Scenario,Expected,Owner +TC-001,REQ-NC-017,Review left idle for 31 minutes,Case returns to the queue,QA +TC-002,REQ-NC-018,Screening call fails twice,Third attempt succeeds,QA +TC-003,REQ-NC-019,Empty name submitted,Rejected with NC-100,QA +TC-004,REQ-NC-017,Review completed in time,Decision is recorded,QA diff --git a/fixtures/documents/sample-design.html b/fixtures/documents/sample-design.html new file mode 100644 index 0000000..bb95d58 --- /dev/null +++ b/fixtures/documents/sample-design.html @@ -0,0 +1,39 @@ +<!doctype html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <title>NameCheck Design Note + + + + +
+

NameCheck Design Note

+

This note describes the internal design of the NameCheck screening service. + All of it is invented for testing.

+ +
+

Interfaces

+

Screening is submitted with POST /name-check and the outcome is read with + GET /name-check/{caseId}.

+
    +
  • Requests are validated before the watch list is consulted.
    • +
  • An empty name is rejected with NC-100.
    • +
+ 
curl -X POST https://localhost/name-check -d '{"name":"example"}'
+
+ +
+

Error codes

+ + + + +
CodeMeaning
NC-100The submitted name was empty
NC-101The watch list was unavailable
+
+ + +

Superseded paragraph kept out of view.

+
+ + diff --git a/fixtures/documents/sample-design.pdf b/fixtures/documents/sample-design.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2ca2eba71442042839f227826f5a3f132d08c257 GIT binary patch literal 1359 zcmcIkO>^2X5WV|X>?Jd#GsLpNrp{zC)9}%Dny(U{n(0BY7f^yFD#--;>$?)*P-qW5 z)Wa~+`swYyxALCc4@T|itQ$nYKs`MN7Z(ub>jgsC=a$da6vBHxMFTMb#&CHVpe!BG z?SDV8!G&K$Y2dNz;i>!oPF$*%mMGqicaDya6hXp=`pNqAu80W2D{gS}<`d4BXhp&I zSX`FlB3#KDeL`1rc!Q>w?5N)jo?~MBu7(V%?95mzLNJ zB-`lD0=M5~zeYK=Gl&^;tgrf*3VLtH=Gl-x^=~D)I6+~UguY=YtudqME z;spk1#8kqqvUm#lGvx1rRNRZG;;fF;_B%bL$Y<5J)&SW%&>ZNo6ko#3scZTDScchWc4 z#dZIG8Q(iF-tBkU8{?CIHr|&UpTkgVrJ+zIn#Q$eL3TydRAI;-TWg6Wyv|U1-%R~f z32Q0jjCUBU7Ixh@*P^R45z|_8E0ly6TzlZAK^wQvc5YWf)|MK{7mK+-({TLB{r7wN zb7AKQf4fBChuY3aTY7$%UEn@!vVoByVV~}@tAD^P zB^sQ^EKXU(V%A|@mV7&PzHhI^O5-G8kOi!HAWf1^0uy+PW#Hd}e2XO{$vu0qD5jZo z7|V`h9lFeixFkL1OEa1l2YgvV6JZy#nu|GVKLQ`bPlSv{gi@BiSwBWT%ZUPKy&r?d Sa&5idC}N%9g2b5WV|X%%#%qO0CEFj)ag1B-+)sYDKcWpdJh{kO~synC-`}-*E_qT@KYo zQDTqZ_&v`XUp-8+YjNW-0SL|fnGFWOi?^KuK9)vqnicQ|xl%1S1en5b$W&cgpq>An zaKQ1ORoTMKweTm}rPS+WzC>Fa+N}kzj~->)`H}rr)hn}xII`+mqm|sUq+mP~KzzlrD+-eM zkk|u+V#|aF#Vz|%tg99l2M(ZN-58+sQqDIDUMjP0_6E#a!A`dAOQXx1uTVTg@r|X0 zRnn97rS#7 zXLrCyY5wS( z)CWh8axF*X8L-Vk(;pL<4N#ziE_&!=fMhMkIKl+SIKdP%fkXHl@^ol+ QsZCdw&=c(HD!tFxA8U37-2eap literal 0 HcmV?d00001 diff --git a/fixtures/documents/sample-openapi.yaml b/fixtures/documents/sample-openapi.yaml new file mode 100644 index 0000000..cecf561 --- /dev/null +++ b/fixtures/documents/sample-openapi.yaml @@ -0,0 +1,64 @@ +openapi: 3.0.3 +info: + title: NameCheck API + version: 2.4.0 + description: Screening API for the invented NameCheck service. +servers: + - url: https://localhost/api +paths: + /name-check: + post: + operationId: submitScreening + summary: Submit a name for screening + description: Returns a case id that the caller polls for the decision. + tags: [screening] + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ScreeningRequest' + responses: + '202': + description: Screening accepted + content: + application/json: + schema: + $ref: '#/components/schemas/ScreeningCase' + '400': + description: The submitted name was empty (NC-100) + /name-check/{caseId}: + parameters: + - name: caseId + in: path + required: true + description: The case identifier returned by submitScreening. + schema: + type: string + get: + operationId: getScreeningResult + summary: Read a screening decision + responses: + '200': + description: The decision + '404': + description: No such case +components: + schemas: + ScreeningRequest: + type: object + required: [name] + properties: + name: + type: string + minLength: 1 + countryCode: + type: string + ScreeningCase: + type: object + properties: + caseId: + type: string + status: + type: string + enum: [pending, cleared, review] diff --git a/fixtures/documents/sample-requirements.docx b/fixtures/documents/sample-requirements.docx new file mode 100644 index 0000000000000000000000000000000000000000..4ac06326aca5b3169dcd28610b4a5077b553b3ef GIT binary patch literal 37670 zcmagFb9`mnwk;gnwr#Uw+qP}nW+fHdwry3cO2xMA7_;}* zS|5FI&LuAe41xjx008mzAq03Ush-II1ORvg0{}q&D$x?MvvoGHb=Fh%us3nip>wyf zZc37q+hsr)y7Z2uNWqccsRs{MmEokZ*N&$!tf^+Ufc+q!MmcIR!!Zy3T}!FeDjaFH z;nh=zYUcT3Eb{<>*yQHopez#!lX(;4I^N!zruTzfB$PylIL1vL9lLkO*_9#74oH94 zFwCW!2#PO+zX^!C?gkJ?O%uP0#5YC?v9LWw3W1L-(I~Hz*j`!jX4Gt+*?fOR-t}W} z99P^IT|dpNM;A*;HOndkracg20fe1K;&dPq0&G*l7($dikKkjle7&3EFsJ)yG_HG2-U-)y>*x|FN|wF}{JryG_a4us1YOd(P4sru;wJ z(t}Kj|LLm@H-G>DpufKL98Ij9=;{7ks}d$9ff*6}FL^~qNlR>-RfG$db%l>)3VC}| zCoYO@eZ`Acy1NvFb=1-N@D49`jZA-K6kz|RF18L*Gt^iN%vF2 zowozC6P0dejV!5Z+p~6yeBx_%7T5RZKZ&q#Q_^)OxdWT*L>lBUYy4I++qx@uKubbAspdIl>ar1};AcOLcg9ec za;e!A_uB_zY*vfy@}B|X{kgx`^%WTMuY||@`ZjWLa<;Qkx3Q-GSEwwUO>7wdr2Jno ziPiE>H45vcGd+J2Ct1XPZRO?9fOp%5VZ$_;OLe(`VUcYv@BT9U0U|ooT2f0z_klq=D+ivAQ zN>A0;fn>xHcAr|u64`fv`CEJ%;X_qMMY^m*mG6D&9q6CJv;Dc2O1=bxeF^{5%l}*W zza>{i*Fp6&pa4Ajii{OT4UrC3Gef8h&ns_iVYl*^eC9$b10d zvvA3_h)6iuLwTKx%HluX9`N5Hgq(&5fM-@Cm@AoyX}GkygA&RCqf?EIq3miiqAh@^ z1_PPdm%$!Yg+soN+G8{ZB9{*ZMp>&RdyzGEn$$RobcRfBODoLA9i zuakz<`CBwyPG~!n6e6h%l!?z!7t6w=oMLtVL0-;>N zofn$f*~7|0Jjzwg#cKh|*%m}PLd4%19-Mzv0d^ux#jgRD}gQ_6P~lK2u$*Q3nwO@-Yd-9$m~)M;*pY5C$v zDpi%mW=BUOM^2%Mw#NaHpi%ba+&g5WdM~1CV<`L+t~2tK`UF}5@MD;D$y_V$0$J4<=oc9j94^OYJz$2rmhTtERM zA@G;Ltk61EZ9-V(T|MVhiFr2u?1GjgQqn>tN zf#ct3^eXOkH-FH9_DEf zY7gD0aEsq}=3dmkBI|-;6#zZ5@w9H!YqgipikVuIDA_h zAn{VpA+6uXqQ~(9TCF!zl6WW8Hc7a{GSOAMtmVF^46aJ>70HQpb!O1N(c@h~W<1|w ze=gpi;r}x+3OyWtWPc?F0Wbgn;(sQFk)5N-Uzw30W4Fox6MFfIDoDr6nUL6o0V4*w zLfA#KUR!)LlR;G%i*;(0pgr>GTc6oVIWr%NqQ^_5SKrQ63f2{HJ9ng_Iw29t??hZ! zt+qVin&meh4=)Y<3j?;I5F)lfwZM!nGQtD@a!25<@UC8 zhVj`j(~vNZ!bZ3mT623LDwQ1L0jh@? zw_!*=AQ1P9LRg2QMm#J2+(W!&-m~bbknONw4s^KG(C6BS4!`dN)n1t!IGtnL97py2 zhXyWm3bap@nx|7^_s(O+g=pa1~D1ONc&|Jj(%<}Nmd zwgwi~be8reX6LF__Q)zIyd7Tbls1zH6tpPd{T)obg|vdyKx`H(#Xr~zX=@xpB@$(j z)5#3n=96W!zP|w6hkf(@tV2CbGO{{f43ke7&|_2q_=@ zU2p1aefhmR^LE{7n$zphHh&~vZ(!yI$qSMJ2H3=1;I6TI+_tIOo})tY43*6p;Hago ze;dFG$QcDutd1Op1Z9RAv84e{jH zfC@kc>M=^(hW)(-;0n-9{Rg14*^KTCP1yI-OW|xn!@$wWA4?UTsW)6n^eQZ@veb_0 zQGMr59`P-NIjSkfHF7(aa6d;WPy+*x+%nARaSTUK;UOfBHwJy83K$V0YS_Upoy+n( z0S)1**2$AOjXDR;>s5$z!oB^}o8@1P_xMDxETiJ(K=s2sh7vSYrZ-kkDl`JWA-9dt zuUZ*C{LIwif_ z=9;65*Qy9B3|X!HC+=v&_ZnoODJoX8Y9)vtT#HX@8?|TQ`@Q$zz)7f<(uN2mMWRD= zua-X-+KiK<=@qi1lD*zZA5yz%apWI}4n&(H5zTC*on}Z^Bo{gM0ud@>hW5lfwTBv8 znB%8|hN7WCAOvuE*Aarp^?}HL1cEZc5P5cPg#l=S+(DUNGrREvuYfIaqQv|}j`TEf z<;#kxfT>ncEUBkI2v!x<#YSa z$41_z;p4{a$+sIkr|WGjIFPR|XRrc2lA*;vL4kO~mlU3oGH4I0{W`fXWSVy&B*9s& z>Ik-aTGrj1Jh&+-g(}ZSv)_`}@?&|;Xv2EL^5Z(44t|By7P%$p)qi#De1{htxh3QD zfOqfVYawt++Esyl85iJIvw&Tb>%^%+;Y+VL6L5P1LrG%XX4c~#z( z<*eRiqKt>>t+f)OZ#hUJ&rvqx8l)Cnb_B65$Wi)!Y`9DvbB;kbKz!%3$4QMdbhb=M{%n-H z?lC)%_69=C$2w+=?ndCX9MBn@@GF0F(^D4!H2RzpzM?+|~GcBsp6Y}NYNM_W6YDuuJi`bD33wpKF)E!aU37hpzEQIw9+i0Z@x zm}y6N7*TCE*H&J3H9nW#Zt4J$M?jloIcvAFq>*-<>z>GkU@u-~h4s5xPd$;-n< z=6r|r$#WgAPwVow=Et76lGFIfuP{8{z1gNyJTLFv;*xCNgXP0)zdWb$Un#o2CaV=S zUC+N(xi>sotPZQ=csFjxHhf*z8NiFT}05gIqGm6%|#T0_l?(VZl;V|jc{W7O$pl!8g>`CPj7@x^;h&j{V2{TtKhBb5( zE8B&oI0!%Hf5kPZG(1XaF2>!xtOVowdCS-|fO-68ZdzaYbAhg+fG3pyBYM5oJk8f`Pf$GE%QfqP0vSxu7qt2GY}-Z<>uQw2j}V&Rh5E(NRF54j`OFpA*Tn27kFd@4Nr0;J1yG~$OA z8~kw_eb4Ow48QivyPnin=)=FZP=8iWe}(@)13#%g>A1><(EUis(AmnPUg%psnmL*# z9FaM(H`28J03#rU$O=IKK%v2NPxun-lR(EG`NL4`T8`yhL>nN=u4f$-%|E-Rl@dfF zuugmITA?Ko2QglHvQ0ZW&bKQaFD^FRMR}@B9%)$2s#c6%*3Z3L@-Q+9R9p-}V8yv; znd4{HyakR{vXFpH$<_o#1=Es2dALfl1K71>n#Y<{yF`4W0!*cd6EaoGf+(wY-bI+I zd^4h#1XDj-3hc0rsASM!pLALvmvuVm6vI*xuc)*PZjM9W`ic=OcIrL{RgAP%^loTa zP9jl&vqpGK{Yr@zVijKo<^eBB1!c`?Jh*2aU_fwEBj=LM8eUXr&@-5E&4AWawX@q* zV_^@Qb4U)_74O~QVxgeQk5qL=ZY5DW8#kipVH=-9)K77rcEYM;(G+GaD+q;`>A zIUz6d8f+RG5cs}wr^fSiAFQJ{GOQ(a@^T$N z+el>$8nG~@In;L?31|yV!1(iR4Etzir{O#(K@23h_+SZJwevxo0YT=T&^;){8 zdp5E@V7ZJnB@f2IQQ_|q**dM{XzKPWx2BMjkBUg01GxLzw;m(8l9BtWyb8hJYusn- zgkEIk(CvIVKR7sL+ou%;B?L?&Y_ZiH)xYQ8NqYbaC&kW81Q5Olqi82jy%HVv!Hd!= zw}auRqU{?<${od549d2uj&5eI3_?r&H}W(QOpn?L)+>{3Wd?-; znj?-&fy{J>=Aw+i>b3DaVyKW*c}tCb087rm@a;^3q+1>yf?|$ZS?{(2N zkc*Cx8cGiEhqD1$+6wmZyn?d4m3p`H;ejw`XZ=Sq8w*|JDiE_%U@dQAbZsDv%Ck=i zVL7<7P0%;Mi4dWR8C%@Qw@->f2d6-5V(IrWkV2`C=hMq7BTi&X5#K#fonE<3u7}eU zs7S7TMS{E+FlqejsbyasbKjblb94v}XgfLtP+ogv+l(_)oGfSXAP_l*X4^fEqb-$( zH3Z!(b6$(xqt1SUbSBvRBs{*JY}0{uc2z%@0JNlZT=}Aif7++#n(Tc zwRWsdC@3e-#U_M0MaRUdk@k>ut`EF{H5OvYu*HNzxv_E*94dCr+;>_4Oao6T(fgGi zm>4NhnECl{ZGCqL%`I`0QIfqFg%+Vo1sWYnm{8amBi#&p)~k`!st#*UGEBNV;A#4el^Qqx-x^kK*LCd>Y&AK5^2LGOcZt*U z%G-@l#=Hkf4qb^{ps!FsPNq4^qMq0KTU)V=lzo^S``e1u zME*F~c8%r(?Vyg4;Sh(JEB0kT=6iWQ;^2t`R?BBz)(7-IN8VS?TqmQiQ_WW>005-F zM_$vflWJuHL+igbuItHtwkr%c!r~&-vTt zUZ=RN;~36sB|7eG;yur)CrAJTAuyeV#DNJpKc}A!KkwdgHCxdAzus9G<3vFvI3 zff)z%MY!lNLBu;SH-#4)zHx;B$Ym|_Df z7-JTH+IWVqDvuh_i17bl(lS*z7-87A!ah33V8jy76E3wfWqyuSq;}#4ISv_{Di6Ar%+Qxesh~NEjEzn&1^ivvH$^qmvh50bK2c_;7PwGSrxOsu zc^sr=J6sH-l*5!?* zTkF7Saip3jBy*0DhYKTis)(wC=FJ!ftw-LiJy7ZBn3ig%%YLu6Nv3V~(DCv1>ihuw z>Gf^N2)*8?0y^8ne1e5LI~966J@Q=6kTLu~{Kj_!7o&by{uzKJlNiWP5ConU#tx2r zG7A-f)1sL0xzQ|g^I977EL({gIP=jSs1?HxC&tgF)*iiyo*Z6VX_$z&=^HX6iHORd z*bE-r@#_{h$$ZVaTgv+Odwt2byva%$osu>t#uf(j7B&WECX96Uwq~a=S5Alf2|I@R`djyR_K-kx z(F7BHU?lq@an3-LbaLn;@sXTk1emH|qPleJwEPNSa%y}jF0zm^0RKdbp9zbw`Z~0U z_=5U(2eAFx=9@TL*qZ%W^i8U-+wO2+bZwUKe;3tmib!MuVQI|gSJjCTC2qO}Aj7CV z$r)v2Si6uqH=3lb%;GDJWR^y99gjiw+vNQ4mVE<#dbbabmj&(kj^$qB)S7 z__h1-m~upR_NWNCjD$2b)9S}>@wxpGH~osZItPkGRZ~Kt&b2~*{d+|`E!vtphCRIh zE_p}&>PO9<%|zLx8Z{2F*YwbwEAE!DTErHkq|y*`h&8xUMQ*_x6C2d-&74c7{JN2< zLuXAmd3f1YBs5e}JE~O(Le1))r~{e&4k(#A9*dg%?Ah=y)mnu<2c^Y4Mq-s-b1{ju ziBrd0!KggpaH+@%LS{TWN<${U8x zk>B9dwr7uj&AASJv(wLqLx>TRvl<4~Mo{^#*jbVzeyZC<-YQ0jGP%K^mHcXd7An_D7jk!B1OkgI5KOP}iHv4r?ocuU$u!mb?WNzx zCtM?bgkd;GVwoyyOx+9GwRbMGEYp#&#fvErb6HLus&XWnaejz{8`i56 zN{qv$+>8+#&*mi8O_L7$D?~RKnO@gHl>^v;0?X+Uf4Vo%CT7QVlZx3JL^Ygrb!d!E z7H49w&&2C*R6Tb!?@e4EuEX+J*UxTf8BrK)>}@!3KzUv5M|YiT=cb!&J}w;^=ESTI zThCqVwwl^pBQyPKOtgjGZ_H=aU=J4~4~Pd#y%!U;0y*Ynji4_5_$bxlQ8L$gyP8)blgco9C&XZl}Q!Ir>ak*Id{n zLtZHQz*zucN@^;$+u5ZO>o)@yhUD-_21IA_p+{+sCBy&r{@D( z(TKTUhi z31mf2W-M1#wko?64oQ5j=p!3hilVM(`nVF2JXHzWV#}T^#;wCyA?{<5?cXCw02y<3 zfP$nC((v|zil|OFUc5nJib*&-F$QC2-@X};$jCA4YK2W8;SDj4+Ote|={n}>S8931 zYvSWfhjq8ABj}v@0i$ZmEt@Htmt$9UbHTgV1;riPaSB z;$%vl&)&5uE0ptN$FkM$UWG`G|Fg`bO4GR}!t!`mrJv)1rmPnJhgVx!)S4FLn!T7j zZTU3!`dyjbG;fo5k0}hk_`td-`rwn*ML782Z)~A4?NB%UAwTDvCS&P~yqAg*KLnTYHJL{lnDpzy;<*W6-*m2$Pw&vKg#iuQAmFnf0Ehc|^%i?^naEX%T+=I~#ck zM1Hrx^e>0|`Lt0AK~d7%QpQCOzb|3YhSi-t!V1DlMDd0MkU%A#pjrejIF@@AXLTL0 zK-T?Vdd!zL|Hq>>@P(Q*C;G2RSOpXS{_knu$=So&*ztNee|+B&7Pfoh@m~M!!{)PnBZS^}1z$J0 zT055Kanw@WURGd#Nf^Uk6{)9ZC{b==b}lJC=A z-EIK!v19l4;NBhcykgW`?=h@oX~4IM7jMAKcR72+OT*T6$sknLAU9?5q?Yf3FY5!1 zSK3kRT7G*ZEa*v(e)aW>l-s7?#yR}>$ob2<568?qpWr0Vx!T6d{Edwlm#80}FE5E7 zncvVS^h3w|^~I^%_^xHcGlnEAxchRjQCH4+Y{u(V(*2ElIxhS8U6ijQt>-(ZC-H4! z#V)0bN4Lky8lSLWoL>n2NB3HHM@!{+(h#GcO~;x~x3<=0_xruji>YD?-hjjAIz9ZR zZ>k8sw|MqB;pbq@bB|$P&G;f&a>JXmW|%KxIw_WO2V~dP23AK3pRY%ha)&qfIWl2Q zMEqSXJZMqa1rpbESOa`mO*P+pt)7ntpASa}eP>6UNeH^%`^VvcsUO7q-TJ$8y7pVG z?{eqnH=ZiD%ZsoS_KR$GxzkS{-AiAd>KxmpPvpVw)NsPr9(`qW)X zSc3}q?NWF~Mf`phe8WoKL1CY8O7HM_Ul-TzyODE?A#x~P<>!dNd1c@HE#jmcidNx$ zawt6cucAI`K80GluU`i=?}L_@pE^$y;=jRpZDm(oOn|VcirY!fH{UzI(eU9X*G#+3 zu4%?AGWCbGj3oZ9f?Kq`A2_bv^r=QM(y%mqOIbSHcW|miD=dA9L%eExT{5^;rnDSP zGkXg9?9{3LHT`-%|6>2>@IE5(aLado$(yMl+N86%a5ZpndwYB_;FQTi9nywVq!#lz zaNu5!mw8K}5(K?Gv;RcucZI(sm(h|EWeGA9ZbcngTQ0)EkN6!>7|)=Psk`^RqxYH^ zJ8JDy-NZZKHK#P%PA!f;%c3*BB{F15oy!&d_59=N*oHb2Z+H&T8 zx`>z~k@wQHLS=MK_)VEGPEuS!yc9|C2XjR*%{~e>p%m?sBt(L2drzg`CJz0R>d(#* zlwXydj2Q)h3f9*-E$?MmshW{AAFt~cUEI}|gN4hHwT#T062H}Lb`lW^*?=V>6f>Ym zLHZ7b9SBbn3xa8yG!zBnSDYycA#qMK=8FBrM-kymD@x-Y z?bkz+|3Q(K0D{~D8sab33yE$`2%UfK`gc*GlThQ7YezS-ZZ?;&<|08io~nz_ua2Z%Nhpzk6!IVOrLE2G-J5=G$XdT`oN$EI7Y-acmPf{|KWL(P=(LjGE{E1% z?_Jt-Y=^uA-UgN?T=UF-yWeU(4bSJ)Y`fPSFY?MA?jMZhZ$HIs^69kP3**Py?$9ms zCuzrCvVO<(;K$5!3uepBT5^=;|@TvP>J_M6v7O?Z%M?e#lP#+9f+aVs!I4{+iE%ykYcoc>3{l zvEe{2aUtGyh_GF7s#V;~E&jt9v!*tV#-MpC@6Jji^tIM!XcRKpi?gHPx8 zhLHa86n~#Ma$oXJxZuwm(Y1TRcD?#wS2C*O`{d(=`olFfa_(pQo+Rp20=p zDfWDlwUEx(NhmcXSHyq|68O9X4j_3OBCU+{_*UgkGUtfId6z>dn?v-(A-MJJFXX8Hs9(ji_O5dMqQ)*;Z`bJ*;h!DT0?t zuUX24p~)~_+k84~0cE|NV`?NOVGDg#xnSM$4U zl*N3uJ)0sQ@2q8RIhS|Gar^K4F;Vxfue&#I7w^=cRjcBzA_R{W9jy+c3hRfVSdr<;GRb`B^g2-(?9Lu#Ydl4c zDPWtqIu(!4akG(P2+?aC*lu+~XfzXh3U7g>SXRcTXLN^~q` zvBxr9FDD>FejlQ8dU$2l80w$87c%Md-~b$CFFnr_20`z%h{mKR?u+<|a6DanlMvag zAGYM9mJv=$xrTPiGoS!y#Q{^fy>$>6jq34{pN?4XJK~1SlNdOxYC7U4 zTxPd^dm}o@x>O+vDq^(^M^MV{@JvjlD?EM(zNpl5CN%vh=Qm6neLIcaWp8V%eP3#k z(ghP%-|5JP?TK8|Ny5;`1iiFC%zX=@T~r$SpmL7NwL=MFd9cIPv`VZ}qin<@@SMP6 zt)%zep}xFs>P7AWQtB>rALb27KxRW;+tSsm{A9?qQj9`RMb&6a@34IphS7KdCv=oG z?93^}9C=*q$t4U5UqBVX56WaJ)Bjm9G$ zpr>=d7T)`@g)mo@g_MMX#RqD>sE)G6T8ah|Ku88W3OgXyQ>B!V6 zDcEsnj7?u?;FA;XSsXz3VxsL?bL37^1tzxRIl_5`6OZa}{a~WEK zR^W?k890zG-wnJumlPDP0f4mv)P;1LZWr9}+fh@n^kK(O%D=GKv{cGE>*dBDNl;sP zwp4jRd`_pqE29ymEsL3t0FvrX#2Dl|+s`kxb@O5>eo*eBH390Lin0`!|Ti z_e_PHmXrN6PYMY`8X7Fd^OAO_E7f0!YYt8>(KIk2Vq5Wb;KmTkz>ZqYZoK=vc8PoS z+QE#!lR$F|EldRPPgg`(KJ}N-f8fhl!tw+O3l30N5dWlOH=Yh1qTntZN>af;YkxrD zDD{H+jqOzhS-{KPwA{5fp7XFZ`OS7@6b zHV{jwDO%8M@3#3tJuh=g+v0j;O^qSCjY&vZAd1R;9s4T*73&G`)P{5F2|{(HD7Hon zO_P6%_J@~)1yi<~Ksz9(6!`M_5fVvZ_5e-7$U+nJmmm#sf< z=WsC z>&bvirnB>~^nS|w(zD~^ox$yr3g3X0_wXe|*kKPaaX&eZS=AGon)hzUU@jTh@T4*y zel`9}BPWhrp2KBYwL9VZvaTjTk8x6H0ycL?FI8K8`6BY1Qm<`8)g+H2A*bm&-B|g0gC$Pow$8fRZ~g_L_)t%n`H!oX`|VE0y8bt5qwQ1H z#>-omejS^oaUr~oU_aj#YomPhcik$yLG*?OW9M^iuQoga65A$qp3BFu9?u#)3YJ&b zAg#)@e8zyU?hXZ>uYpdu;vXN{xpI%#u|dQ!~Dq8 z4?vsK0*P;#%VuYx{C z{_MGv*Fg0UUEfV0yD))e-w1-qTb3DNW$V>mBRAwx3&S_mx$iNLV!90UobK3MOOm_v z$q727j%>Jcrp1&tf&!7<_iKYUoHKVpI{|HC(c?vqVrZm}CUNX{Ub@*01ChC%np%!v zl>{P>s@#X!kh|*O*M8x2#Q>RcW!E9kRFNzJ-mzh}PEG9aP;dmO!ATv-jBx7qV$U47 z18M|sjMfKlL<5rRZppksB4=`>RrrY5l|m^B+YvL~u`wZbEe}L6XEh+tzHcf_1MbC^td$BpyfdWCqVv|DYMA+oGCE>k_p~{o|$@!_!Gd z*slAW4lHwJm6qNGpbK=j?9`sA3R-R}X#mM{-XRwK3apV+mme*Uy`CnI-KbY$>p3=M zQ$8(+`8E10L?kgA9N>JSFI#w@6hQM-a;?Lc6$RYo2`kGrcFgf}V0;Vug$`RO#FPSW ziA#68j`#bw2A^oAl3=uBg;LvtNevdB6`j^l?7+%A=bLzT)fbAXLj9d1ThXR4wAvC- z^a+#c&6p7Vs&~>6(;CXQse-AhO_u>1GD`fsa#iZ>;WkdHlg&s+%#`Qaa6{%iqXUX{e%piT0%23IWQNr=KMBY*p?m zO|cF$=0;r@KehQG+I2N78tkd^ns~In2C;X)z*H7h%5Ju3Js!tfpbO=m=TE*M1x=Oe#3wnR!DOjXH-${s)r@Poq=gA%tSaq7Gs)Q@t#aUshRdk1Wuh`Ylg4 zUFgofEPd%d;oRQ3B+%2Nt8uOreM3A-zQ&Rdrz^uEpfg+3*#+_bpfcTtao%h}3XmS< zD%jHkZg!yPM5N1@n`Bj@V8aiWO%A6Ky9WfcEd_N7#IZ~uVl=fa+po0XBb18@(1DZR z1LEt5gb{16b;X)jl*-5Lv;OV!DaL?7%lxltnwtjO7fj2l>ikX1JG6V>3gylMU#oZp z0g-Rp4x~muUIr8)2lkE*&~BX}?V2=2v#Qw33dJTvDRJs=gQ|xZsV)?*STSK16t$>V zB~T2$twMW$aydSj``@X^jQi|iS1Izu_*8e#;Xh}R@v4a-#xMEu>VO2dR$0#ZJu!+>!a z2>*dHe(P=EaebUPQ;~`C|2Ga1!QVJ;SaR(U5PUIVk^Z0=O6Z-x+WPHfDQ78t(wv2(`d z^s+a8DK8e`zll*T%O2)c(?g8+mPO8~D1aE{&Sdyfg)b{E(i=E=Y)|yxXmCn}|3sTE zEBOCH6BZbT?u_7Ti_PQ_s-#kG0{) zM@BpuD7Q*Puq-?0$^?ozo&&lpYuyqsuf|FXlwj8=dC&6DhUEp6Fnzr3Er{y(EBEI@YUJ%X(* zCW~&oHt~OHyf6ijH;4B|53lE|PH(8oT6dPK=lza7mk_4_das(YH#h2Mxctq|rUtQ~ z9HriG0877$O#CI!=M)K&+$;!d?}HKyOLxfCw#9F=gpbjIL9gX>@8iZ7Q>Me-wIB*E zPIV{*u78_1t;rK{-`jX;Z*e?_t%Cu>EiZ>6!s(3? zxEqJrx@$6}@DP%yh%iGd4t8;8%KSkxj$A%AX7U}V4n;VbDSH|W`obVd9T{90%J{^8 z6i^xufsvq@h(7#DQtb&!#FYEH20F+nE_5ajL;LhZ2+lfy7&+Md)AD7z1IDCm=TgED z3cT)N9mdo3>7XIhvOQw-p^|a58c|(@y431gjK~RBq6RvIc(-Dw22bp-UWp<@Q@oEM+tQ2S+WOp*x!7v9o@8`Ri4l6$lmAZ5| z{}c{#b|fc|kXZq1i6S3<89B)Q%ZT%T8(GuYm9eLgPf27Za+rNV4I10%2ifgpst=P- zS!9mQ>?U!T(Uy~b)i9FqJgx1y9VtY@pm<#v#z*+o8N znKC}QmdtEGTSWS*U$%)C$W-E6r*@k98H7!9AA}w5^jCS)50Zw6vCf0%LC(5pxy*Tv z#Jmw|hbc{EsXt6w1i&0*-<@kFIH}{7YZu~{3I5Fx=A7ZO6nH1o{9g=f|2M;w!%RNw z*f-Or*(fMjOR>p#sbryju6S~|BwK-OSJKH;%IN|sJSbNP)uv*uQA6Zi{J{4M=jgYK zzh>y^&x;R}=gs%+cbe#a@g1&U@rGT~G~ZddYLOZ~F*{>lN0)C!bkpo|*d{Dmi9P|G zJd+RQv%3uT#YS;1_-L|@OSF~~obgqUTw>fdwv6GvP7HV6VaHZYS*%OLF+hxBDg|%M z5Av4@as(jg!FyciWbjFrP0-L6KMf$mG2n!QY1V8g=1kA-Fk>-F-B`qTmUXvYC(4U$ zb3yJ^7xf`qQXJ8ed$B2zY5GD)+f1X=)El=5bvbGJkOmAQ;zK%MrB1f98Pk#>P!@^&GK__FtN&SIV{|h?# z3;LV-U#st_`)1o(klVkYiM76;QNsFnn;&*KGW23`h+?i)dNXG2apJ%-LR7zF}Zz0Nx@EV;gkPbWly%Q z7nuGHUL*7cp23Ci7kEtG7x?!7Q#a7GTj$$B@_&J&f=T^MY42BSZI8*1M=MU`<*VDinX3VK{pA9w>X<1|9c~aVez0qBM?(>we7+hUOzIZMXf^EQkUw76_ zB~3PL6#ye$iXK?B0Ubc00Zx(DYijzeQB#FGM^HO|sf!4vz^rkL4WDsBVwT__cPCBk zNv6-O#Jb5LkeFT0b#fD`)cEk4U@kj%qJ}Wbf*@1z)Gyt|4m11p=O>nc1GpO}JuO^f z{@jwyE+?3Li~0k0Jg!BFd6M01^K?-bnO@go%Wj8sG9DF|g?-o(i}Oc|xy<6l8iITa zf^^jl>T$)(BoUS>U>2F&<(Rj%B~@@-grj3Odd490^vdU$Fj<#5ma$gy^kPIATq+c@ZzLLxW)x9a3q;i02yTS4^qCzrbPu95YLx& ztyzL}GArNRJf$oa8K#SfVhiDU5NosY1MCuuIVBY_oelf+U%vQqlNOrqP(I}EHqzzHdsq6?!QE)GsygTN;Y_Yh0#?y z^V?w!hMGBImm}yXMP`i-ecF}}zIUJ30yCGsys(JBymwB}k+V{lv1gHbSiYG&!%XGn z11fQR>W1OevLFV({Opd+=Jfni#_&LH;!ehT@;yJr~4g$!O~ zHVTk8cV29oR$7v}89-qu5PbeU00bam7zpBCP>FmH4*{M&NXsg6?r0Q!JR$XfW0QvK zZ&!m|l`@U~o)wnv0joYNQRrLiVvzJGUTL7aU(8-}4grAAo%U(+P{*EE@i#}5NR z|K)85^p7`aOP(37X!I{{?g0(#=4{z*{kp%%boskfSUY=df>MT{uh`s!(V-0ZTF|2) z&02~lVZ0!8{lx>eK*MnWbp68tw?SJ8BsvM`OX=MO$}r`^u7M_uAn0p*&TqciH6uLK zvt0w7RF!Xo+VU0C==whz5p~L2_oGXjI>E`!t10Mr@Ei?L|?FACt z1$5=*t^zgH!r^v7Q%4YV^*xurrNVPokZSw~Ihcema>E3{=#hFtvS`5524U~vm=C%4 z&;XQwE}ZdS7slFqXnleDQ}ja6dr*1`8@USxU4IXlZP15&|L^QK%bT8cot2CC(5Wru zq3EsL3k~jo-9lMcf(|{a&b$~-{Tn8sJmzY+{7~j%9Q<&`1El{Wc&~eZ2k+eEcF1}- zJNR#g7`haChY)%j92Q`B^1vBffHFfu0y9$k@d%}goD={l%CV3@gnPJWVS41fAq)!qRA>rY8!*@_cIUasmv8%|NdRA6|a@Vd6TBvcEPtzMh_dJP_3V(L-f2IX12pEWv zhyM8n0tU*eScZo-$6n;Ate$}~>tNE6uNl`~xjjkKvkU6WLeN&=<^K%g|Sc z1&EOkr$c>XEhDPr9xYAp6*gfZEr&`HFbt14ZeO$bD^mEZSm0KbCrUEcI$lzU5jsL(jpgMpNILDB+`#8i$6Q#Kbg+z0^ca zkSM`Hm|;q3nEwc!3*)?J)I`m@s@utg9BPsONC-Vpwa#fVbI*-wV6rE(fT1t0zHef{ zX`yR|DbYdz6JLM;;e?#NI>0S52I}Bj@JN!eZ_-!jo||S;Yhe$}o&|cV?d!7>QgZ;e zl2i52WB4cF!=7V0d(D8oOf~JkKR0bQDrY^H<)d7+>Vuc*{RP!ro9wBj{uk7gce8ry z83e9CT260nu8;5HfZ)TkN*vVcAvss|8E;Ry!)yOrx*g-{6fYlpDij|nBOh=tEn^QI zRzNC#q)unrC3v&Drkzuc-syP>;=?oLAD}BA-zKZ||3lb2Kv&Z2U&FC&I}_WsjfrjB zb~3T;WMbR4C(gv4Sd&bg@8o%&d*6HiFTQow>a|XH)&A|he|_pyS65fN9I0>ZtSj4J z10L!EonpXJ&<#f|pih2+hJ&%<)Z?Qq?)A7Yt8?2c+@Ekmti@@in5qpm+qLyMK88dn z1)K5uAF1+qUo=kZNVn`3J6d@c0{NGJJdzh$94SvJS69LIXIbnbzoQG~H21$;<}}hr zaDRh`D^xHN5ur?89jgN)XKArXM8WQ)aH$9#L}uq=Y@!O6C|p)1Bl;57b%=#>$GaFo zi3&&Sbj^bg{3}~aVMX1QH1l%ozS0`G`q|HxUTaUH!f*-cPJG7y`C0I*FW8C~H-;vX zobFe-6Z&7wzI#fpB-OD^#Isg8{YBB$=(NJW9Sp=2el-zjzcmq1-zWv;qv(&y_*7?q zD}tTp5cpL-7S8znSUz`b1SsX{_r0G=N!J#q0Q$2m-S2{9LEr8GY2UJu5~IBGg}4e_ zOSeqyC$%%19`8ebRYG-|#}vYOvr58wac}`?WYz04kN(<}qB)oTmm^3%^jD-W_ZxvC zYgqX~c|BY96Ui&M$E87e<=_Ip@ycp375fpFR#mS7GZ2+7`-)eHQxCMO8=o%GUlo0I zs&~}u-PPZ0}B)gfo^O&ty!F0HMGBFT$^~bdP@%hVcUs z2Er&2Y3|LuNu#%>R-U~-$|XjEZ8B+ zc1mmiWHBG!k35y{m|OKD@YNy0>c^9_w-Q;;Zbv`r#|*7&k=ej!FJORb zz^p<6l~Te(0zXpwgT}KF+Dk$3UW8{gfc7%}8ntWZVHiJp^j82idw?Z``}YvAeu2k4 z+4hVYqd6-53bpyAL4*Uv1kt56gGT~@lB^H^O6LE`n+DuzOqLm8B@B0fzmUF_XW#;= zglSZdo|A)VgzTd#1rwOv4w$_lwC6F9(`XNgYZ{(k?CaQ9`$1-j-p9GFpW-b6&k*7q zbm-B_d=r_Rc z{56?79(g-$*U)ZYqtLPL$g!QNhZ^h@Kv1!*4!J>q5r0xUq@ewg^&$a>E-KRpkl1}{ z72sG*#AMn&$7{nxu+bN=(a_m6VorI7t|*6%o6Uk@y9WD4C+_~6On7F%SU;UpG;IG( z&FcREi1Vff04OQ^4ba((ibuh@CB|t7`QR}0KC)nV=n}NghHHYxxo{)9K?F7$(<2V1FkV3o7;xbS&&2GxC2J;6#79Oh*L*8`aj};m_t=Xm1mO zip8~$d+)m`NnV5HjIERf=hU-F(E-$EAOAZ%Pbu=>;ra1-_JJ7y;yDmUeb;s2R@oRz z{G$I=O42c;J#X(hX?7qBvwAz{hlm_IR&|{xnQXMn?Fi zM#}@5(Q7%}5mDt%_1R9&Dg%d3-8NOgps0ik<2V-4HvF2jJ_+8oc}deFcYSxE_9+?} zyfifI?1%Nt9{wsureErq0sog(vKXO-G167C#%dU{KC{QJ{d8t-My5j(+Wfl2SbTZFr3t2Fy08 z>zO^()XwcEjZLrFTZbm%H8pG8znXzKoCw*_w@nlG(h0c-05uR zM!Bro=B3c_&Srhvw&J2uF=rf3?c;?2y@Hd4gUINI+Hv=UAO)%>f)mH6FNl$b6E}x2 z{_F+FnWpi&#P*Lddl@lq8Zyod!J=mWz z`=^4R=8REAmaFIgNZ(dftzu`6PVB%JeoG;4Y|vS0odP=$2Iy^cHk(7Hxi5r=T?_uo z2XG`tGvf-}t}i#Fp_-FtbOOvMO+!;cGlR{|+}KSlbuA!OBSX|{nu>#iT!((83$oG= z6qN>$NDuEio!Ps!tVB&zqe|~O@vSlQB;Dm|Ktjw-haq-()gZ1=<*VQ_QlK4BJk`Dc zVg6S}9|QLrDW=XjEC9|ne1$PG-83|Hc0WK^or_uvPyf$|lM^_MjP=YpRe>WDLG8y= zYdK6Bs76^;wCN0H+0OHtt$QU;}aG_aMc4qfsuzqzTi2Ut za9F`!AZ+02Y)|i3=_(wWsAg|j26S~Nu2?{*Sn#x^>l+MXIP{46%{Q-EByU8=rF!m(9}q(b70gSfgHQ%$;O8jY6Syhu$(Vc>8)B6s+5+$6T)Dw z=?THm0spL8zc%JL%8)dSz5DpThqMd#xRu3s7Tn6=fuTcD{{lg;il~A;u25T{p7If> z8@6wjFH>xI;e|)v(Vm5-{tZIiRF~6cuh{SQWh+{j!?cC7{9i0|J)r((0hW8^pcT{L z^tI9&o!5WDa%ZCJt4rB+?cmWjMC3j7g0a3YCW_0+B|Vo^h@MKDA!O9v;er9hHcqJe z30NOcTWkiX9UEv2n3P`pj&JZ>A%twD@>@k;;w|X~ZWgz)g!3fCtX~S`DZZ%2^e(?Ns#sW$@WNxg}-J>;*sPX_AH*bQT#<@1^VZ zLBaFV;z*~*91~+>J`Aus0aTs> zL8ul#rf`Tu0r$q)aWR-$Yj9%Fa6B}+jEjQTJVx{krf&z>6aE3J{tF1^-$0E3pn`Yf z%X<-+dx1uANyBq_I_(3)&VauV_BHtcZHu>ImL%}K8hV28xcnHaa==vxS+SsGfm(5X zlX@9nvEqm0{4;Urr;cJqnCz4EZrsqd{Q-;OxxT?Yd~A)ElV4s02Q?9B((J+|2_QbC zWW?aYFs>xv&DFvefO>J9ONf6f0Qcr}EIF?oPR~+`7KZ^ivBe$wQ{jB+&cRTuSg`?- zIRanFW{=~tMRoP<45E-$n>IF@T53#F@ zICQc%_T$n2zaNw{@C;huVgmwlu7v=?`S1Dp>Xt4RVrFKh#x8$P)%Vp+D-v(GczHr^ zJ{?xbO$8zzpY)wU0U6Po+`q|qc;v6FT-431Nj#ONZUxNN7t9Ml8Hhao`?C%A!Txw;*Y3Bamnb-%yo z*{Ij^aWs2)nf38>_3`|E=HB_1H#@nhd;4;K*;!kQcy*Qc0GPtR)xFNgOQ(;~+1p)3 z(2})zwD4|^n3d+9rI4qFIO(p>u|DcPW5CJz+PiXbWZ1q=K5`Uwe-*X)R^FZluW!u| z6@K^O3K1-I`^U=Ha{nQgy>`;o>2|^FrD2uA-%fy{ch%3+`;Fc5&b40OSBJ0m!N0`z zDb38!-~XdU*WcZ0e|^%oxc7;^J@CrU-;-aztGwgEqp0+Q>%(Z$IcYR6BrCi%?a<4= zt=oP5(AjA)>;BU6-s^&6Z84*^*(|(lPYLwKD27y@0m{-h0o} zt|S~=Z_UH(E;i@p03`a^JzlGDtm>j%eK_XUxltoOQYJLLy$ z<4H#-8!?7jg{Vy{b^912PCq%#tu=;Rm&-fy)=WcWV06xcMlyB>+(8jxGJA~vYe>09(4 zkM=a|T4p(Bvo1x0ip}Qfq1}yI`KAehkrxhEkm^e zy=CI|lJ9*M`d97GLLSq$z<4EPpD0bA@PDEFXMz7j`9HJ3VrvC=NH6>e!YTX14j2!@ zvK0s+I0OL3_Fr88q4X2vln&V*Pz)MS%=({VefD;+JB(t@pE#Q!oN)7>IR9DTKQz~g zH-Rr|lWYNsp#X|~N<*-As+4+0c(}WbeF;4zKGgp5S}T>*1?1nMKd8~T zQUV^6ep_ywZqJw8?LHsJ*7K*<4{X(=`y8;P?vsd^?~fi&P!~?#CvO`$J?OkSom(#_ z^fizzohFgFH3V>b?E_JTE*Y}<2)~K2&C>py?|Mk+Q*j!ma4-mD3J2<5qs!KS(+1_{e)lpdWp!g!&?z{CrO$u;Tzs) z&Y$gE=fcauAGv^c?t!-evQ!IEQHxyWw;jmMOjx#IZCy!3pdRc|g*>$16TEtb6wxj8 zTnn}5J0GJC_0b`HCkj3Um8(kXQHb21)I;iF!ymL18YZ+WDsF)_C+w*Oe^@R~sqlhu z`0Yt?xdY2u>`eruW>|z70X;B7CANIB^dTHhW$H;WF{T2hOT_V7P2dN=T)*lKYdD(R zeSJyipzG3|yxZP|0RM4**6Z6ocaEpNLVwSG67PHc7`&^~n$OegYn&K=)}h~>*4x7V zM8j*7fF3)ZYz6&m+u9We10J0nqW!}nYewwXn;wNxkDYcu@29qnT%&~Vj)Nk0sfqQx zU+fA9#51Z7j;Fbf)z+$WXYXb(o55edz@8d5DR3!vPZG9rDYBf^_3hnWx5zlSMj&HK zFk;@CxXmCQN1W*za?qZ)Ont9#^2T?vuV%}!p0P7=g;GIck>@kRbDHAh86`nT)MfVS>E(kPa4bvuIn!#x>i%Q(4c~4<6-R=u~VRyjsA!D|MfE zm+mg^+)vHzNp{AXG48#aS#kIDruExF@W38`Mmt-q_tfUi^N6{z(?J*+bZp?;a?yDf zmET4_J@C{+HUs9&dGq^m0`aJ~*;Be$PRS z8Of28+Mtz>U8kQ=$z3{%eUqY;(fyRi)!AxFM%1+KjfFmo0kAtKbztd!%D}=+Q?(=4 zs%lWG-FIWzV8HWUjjOH@fcf8aOR+WB;u#K9Cd7yOCw9a;UM4h4r$iJbnflK+llgT* zOSjA!JuvKX{sbuVES*$*-c#zc+ahaRE_U7gSOa%IO?w#38x1+3pIjRX2EF2P%zP*K z%v}}c=@ion_Agg&5mW=^$8k?rC@?%Fa_$50t_nCn^P+HFpZyAn3)r?1U326R>u`2u z1qP(`uzLBM#%T7dC=NX6x~jn=h45=SPwI3EaZi(e-FB;Su>SU_N;rAEUNz`VZA^A@ zDioFa)t_nl`bbr@c$eN8UmzEut;6QLY1x$A`nX_H zGY1zDg{qC;8$Z0Ewp0V_i}}(9pPu0cTFr@4=^;UN z=Fd_BHrue3-(~Asaqm7Jrr!%i6fiZGa*9$r8?7b?Yc2dHcvQA?Qc-g<3GTHoe^gNU zpZ*vr?b3>i)js>-Ise=0O$`N{%jM({rLm)F!qsH{KFnjGy|d!z?Fnb!tVgkwy?Dlq zr`hUMpL?lkoMEvXa$;Ms_TBiVZZkP*i{a!11$ysR1wTkhPX7{<(7oN2mCNDX@n_%( zOOKt~c#FV{jY^6-Lin14^ZoaIo3V~t6{8NA%?&ApJKeQkbbJlITbi0k(>hiR?=toOcpl z9scFhUi?#LOltUo6iI2;O~X|g1g2Mmn~te!Jz_VuMwJtDvU-Bylf@r%|ScM zVG>DtX*>7S?2gB%?3$}C9mFts9yf6E>THNTEuR1rYf}GnE2oJ&U@LRzty~%CTb-p6 zdVcoR#rJal{)v9gCJ*&gf2-iG-(n}l#o|5bv~|Ew66QK{tB1}^banG*JN1*Isvmu= zdG>sVJi4bOvuPfb#E7lE$wY?+Tgs%s@v)c6l;0FaG4+L{I5UHie^(A|X6p%k>6+i6 z7spBB&GpaHlY}M8`(a4S)@YzU$j-NLskweERV8Fl&VvfxUqj8B8pzTmdwvk`I;D>_gtC!O|6&v zm$iCRk7_>TnwKorJ%jAPKDuUN1iX%nyai1@?Z|=TPQ|LV^x@&%_}lhC0IoE{m-N$S z@G;WT2SdND{e*R;)l5o+9=`dBI7(8y=+$fHhydSf!%*#`W$XR|CjKV_VN5o0fR!uC z@acTb$GL>jI`XbW5ZYd9w{wq`hgTaR8!`0g0g=OC31%g$+P&MWhpQHT4!kRQOVE&l zuJ@odQ3j!p+U??Ihr#naXHE^g?-=|V^0hguMc!6lfmzSG9oaVw#CZl z2((2h4%PwvElf}=B`a0jE`ywrFjXC6OFJ7Hyl`Nba+ZFmVcseUN5#08(~Hi&D>UBg zMZK5s$J&hZos3u7><;!jTZ$rTxXoytpp#|f8@L8_wr~U8Cv3H`fc5Y7H8oS!kK+cq z>s_#*HnQergW>*^kn9?8Y-H67xNdNHb2xaPZ^;G|VaZKCi*0uA+7MpCZ{RwOSrw%k zxf@_Nhj@d+wZsUual9QZ`Jsi9eyMeHme}3W;0ADuq1f3nVTp5_@5Ub9a^W?;f>_Cy z|JcYH@B3pWui0DH$&r0A*d6FCf}m-;_?OgF2m8OI{wZ1M;*K7aVl}+AVsJ;hjQKt& zxU&@vwvlDSz98Shap|xp2l`;=;Hvm?yyekv!~f_1pPf8w)LowYh7(&G2L>W&p!Ys( z8^;=qQjUu~$x0rgbRoxzT%rG^9B=Nk-Z;G`EeTB{taA(EABanME|X>@nR=eOXsv;6 zV2Je*qKz!QyHn2SuDbNPN)s4eXd2xv7NXv)kvVP+$x2e2N0At5b=fAjY*Y!ZIP@xygseB z=Kgr`FhwqLd4(wj0kh#Ch4nlI%?@2&H>#c{s$Paqss%n*KJE;&T=%a5p2m5eK|Dqm zgrx@ynckXntg8#<1+Y#XR)pg%BTeqR?(`Kp}>1qtwe?z z-kRE7b)&kyw=f$oy4RqB=sB@l&300aew{j74P##oZ7;_k-Wm(H34aV!s^_v>t$9*y zh=9cn`SW2JC^|K;;I#T{iNvgzHCp?R-D*#0$aM-fQn2F%Dk*o*0vhAaKtLyJ*?Lz zaTJE=Z%7)jEw@Zd2^z)*wXJ`5HuG-XX4nHL?pBYRYpf&)bnn7lEJo3HvC(w1O#zD> z3$AbB1pHuTRPAn_IJnoek0Ej9>uNqXjX6%i%<~H~sZPuBl8i*Oua>f}4ruiu(FaiU zu)c-fcP$RK=iKGJFNC$9mbQm??wjM9YiWfrXo7>IvcI@mUouWfbK|VN%r-?$>f;jO zeotMvA;e)&J)T>5wa3W}m$A&1X^-JJd!x7Y{!^#*mWiU6?b7xKfBz@YHwN>q3B~AuN_o(D}qa+5Wk| zpkh>}zRyxI6s@#bl?TRydZ&{&L{l7u3;T_DLg>;r(p;=U3|Nzlg>eXNVGe$uvSW2( zrFxp+LY@%_YEjr3=a5k?pliAhNYGAT%rzW}S|-v=1PqN_44Gt1FD$VzEE&!8>7)Y= zik^TnGE@5ziKI>!wXU>6*D|6nyTMHhe8jDSo=zs`~FL^T4a zZw?NQ>7F%Ggu?{WWt{LrKH@(^A_4y^|4!WQez{|T85fkYx0(T5<{ zm#aMSofyQ~ipm-tYn|#o21zJNC7nUkU>0oz&dv~mRMg;xG#FvEGu4G!6v)5=A}rtQ zR6>appthpa?$W|t7)&r9BXfZ=ydLDhHVg{L=|!l9M|9^V465OoPZ(?@1nM!WdveMl zVcGPkRq_R;QM=f)94W93xPLf!0Z14O3VF2v+F=%JlSb_#^D7Ps5=(*e+B`MmZLKX7 zvWp4KT`1If#z1;M-KPh%ekOCk-Ybdnp-@0@(|t%P3WJ>4qDfKJRbHouv2GNiKwwq{ zTro~qG=f`Z5kN$(ga;iWV5D0}{ZPaps3Kr`j9IfXR%&rDenFztOo7YaQNBSpF(ir0 z=6kklfek>2HYVR-3q}ADgYd8dUt$+|4_D|mPFN#DWz4nPSfXP+t+0ncaWjWCKz~oDKh(nk6kTp!R#B6?~Fw zPZ2PFVX!Y@PU||u z@q24a1$XQ>jE{hrz)gaIpU6u3hB~30^ztA$K`;Px6bekudo>t17?9VGO!!WN@G?3) zm^QmUtvwKam%8rVsNlrS+SSC`)5hA}B{>luDg#w^w{xu@$W3^qpFxU?j>7mgc{KX` zHF}Y1td->30&a>bU3*&l6VcnK&QZf?GPL$j1JzVyEvAy?W!>Q);y)ibj>{4HSo)lUgDEh#x)3A9{Y$tUY5*byKXpb66r< zT^`JEHF5Fq3H=N9BMYM&=hGV@quU*$gdK)>V&t%T50pNBcV5VP{yzs}(^vA|BoXy` zne;w*2_W|qja20iA;_CgXfl%4Zd2f-Z62i`io5(YqpqKK?>YAFCl$VpU}j*XW*lMS z82?)LVZ`yUsohpGx4M4j>hhbK-kOs0c&qk)Q_A+Jk{=IB$U%^N)XTiD8W&&n0x7fh{r-5{}hpS1#DxltQB29PoIf-yp1vo3+sJ+KSk>DOX_;R zG2w^edw^i8*vKzt_a%2$CcH1lN`5L<7|z{re@n#Jy$9e8ou%cW#4s?^(Xvc4u#U{N z3MUNsxXOCG0bEPZ>dwfzeqR6cTwUvFJJ4e2uVfn4{hm1AZK@|A2CVP#wl--2^C%zO z-)0hgOT$4~E~h`9t?G+_9?X2mt$IF=4xHNbf6;S_a0si{|KED-{jp_8Tj0;m$h3al z-@WLJ1tbS|9;#=PO3GknV{D+KVoEJb0;|W+`{4tr;v0Dh%p`94m%i|17k^MM zpPgqc!s?XtTFSLBzPB$=F81dGV_7um?K;0TbE>7tHu>fPp;6`w%Hg9{IGiY|wT0-U zcEdSgA25%ZzUk}^V;RIrLklxDVwNCH5v8xSzq$e`=fpewj%5{PaULR%FuMl=%?PuA zK*J)>bnMX**CfIy)O=u$FhiOq%Di_D#WW7!s~Cd*F{PI-%o=QAc6_@6Ea@sd@PlU_ zX?hAdjVN;o6wU;v5L?k8*n4T&F@@&{ z=wtBp36>CN0)vZxGwVsWaHDbu@cE1n!AhKI7`jp{9$nfq`au2)GDV#EZ4a#w8~blD zPXAXi7V4}omd%w^85w#)%3oMsw6u6mH3%BGt}!dL^7`^_2r}ip(p6HygsG6EReFst>E@zz?r7cfi>SM)zK;P*O$Hzg-s*x#SWQ zgaeTx)FP_%e6HPT#V4flK3`e#`gq@pnRXX#=8Ff(JER8pKWI*N-4Swt#mp3UfOX|m z@O4m$Q1R;DvmqqIBx^;^4?Ln~#=G=uu^|(31r8DaW%n!05se8{uLVUfCazfCvl6!b zh>AfBU0rc&AhbkjOBlU+qhSw8QE%o!$WzD@7Z``KmSkb~(tJ`JB^guDBIH)waR6y; zZRl3wqM!?hP%7tEEzNLJMK2P^Cci5bR<5Wk6bGx_beKH9EBhk&Ht3cegjG>btYmO= zHZ_Kfk|AIY?(n{d5|8Prz?$Z8>5t@Ta%)F~bndOPsv__qosI?0A)QIx3>^&t@Zq5i zVKQd&r;1x@X8gZZcmju*>DfdKN~ylS-@YokDgUEGCI1+pe?dHy0jxm%e^@bMcn~^#0C61NkaTP1L*_X#d9#Ky zykga32-nKz2jo^A|3;f-HJ3SM#U&lxWNX_F*4aC=d4;Awwf4rgT`$irdGs~Wf;$(7 z*B;o)O5X6za|gRdI@m_Zq;Ckx*cRu+P5A?tdDdDYhCuBN%Q;M9Z7lvw{wmy3kNaTk zhyVIn%Yi7Fk|Yz<_q7wMuLu{Yo)nHRdq9cGlV~nj?^78Frbn_+EJ$XO4Q`Q*cp^?g z6Ny&Ec%g!W-{zD9#cA6M(32a^P-0f&g^B|ui@-JBuv{BWaZ(nW(oeq!BNT+)bX2je z`S8#!H8H{=S#uICCTa}c-NKp9m`AW7Sxer^MLFY(G-ysHT9ZwNX?&vAi__B8qBk|1 z8GWKw1#0|*I*gN2eEJt^eON;Od?xSj91LqMEQkm;JR~crs@*cR9Yr;o01eRkuwvyP z&K0I&gkais@eIfawrnUmbn79M32h;oPWt5tc+PORw^10n_xgRDip9FZbiAEf4{ zTZN3v{Hdr3sFJYAu0^9rc??FS+hvY;nX#w0zT`o@bT|_u4A=-BGDHi-nv;5nD;a%{ zXj!m|xLls%X1o=<`O0LRP5MlzYFE_4a|Wk?L{DvQq9%&XNCW~9JO;zy-gt2u!&+6E z(WoK~fqMuiOxi$hX~HgwZ5k1D2nPYGnQYl%k=>QjI!hQ}nuxOc*X%Pa<>s%cIz^PU zLpbswsgZ0(hZNQMDY}@}Lt$`6@IyFseW#;kS#--~Xow@K4EQb)Y|(nHC0kDai!2VB z^?Q#lpPG80bS)9pJFA^;)=75ZF8sW|&DUd}YfkrIPDsXbgV-*n@-&r$>G2_q!)kQ3 z+pzPu<&bXv3O>ZAW(jBA;eiVJYIa}R-y1%oYAMMs(3(XZqA|oz=ZR|`Vm_~CVbzXB ziTntE4l?29{j-y-H*#C&-a)DA1Zy($CDIOsdQjBJhA7 zSiA_ixAO1!zCt3Qk*uXD1QmE$pLF}aT`=sS6!>5rZ zo)wZx32>@R-q>>ZwH|oUGERitOHui%x^U>u$|*gna1+T^O7#?C=+Lw<%3_*P`H|0+ zQUTOv*COKR=to#(g$b%m(dPRZvwZzF`H`RP7q!}o$heAFq!(p=bk3o>*ax zTz|kC1E~Qmfop|EP7I#{I#ok`Ftr&&5fvxqXHTv7{3}C{GPsWFlvGNw3KOM@isbNr zqZUK|H!2ZA7&+l$-IsKD)qgC|sRLM$0E_Wy!E|S2(i73|)u^2^FXUt|b*16##m-1- z2-B^vs`V6qY6?5Vez>EP|HXG7{1bVzGxF9MPW6Xm1t)H3QN&vqq;=4mAUYd=BKOy- z8PT-TNVJS!g6NHDKMw4(Ac7soNq(sVC1786INfX>BlUho)cI?v4_8f#?CMK3{^oryuw^lCdNWIcwI?b&)($rfH zNsgXRVR=`TppBtQ=P@CJcaiX>BN~#N@eUJr=?LPWwI)iAFwsIJOVxsB${DucKhva4 zXy&?fwzLdE3oWgl-~RpPxX>V{`n>^S96v@w zJ=dX%qx7jX3~3WN`KmbkCW!ADTDmu7j$EBPC6h;6PxA%IUrW zL{FL&c#5`a7l<)ZnuDlTT7rY<&$M_4(af|&haq?6h^=g*j9iJTt8_irPl7 z3644sSdufR%*(1U1XZG59p$^p|L!x7$AAS+g9w@|zd>M#vBIXMGIV&>KXHN5r{j!{ z$?;rhk&xgmJ7MTm#P^K{zb*Vq(0C_8- zVQW)2)$tsbDgsw3+B~GlD3znNY>vv@PssSuT_fd%$*rd0N}}Na*T}LAR$lGQl8SU} ztDwnElU^S-r2+F}ewk{0$fW{^dw0Mf==Qs}{t1;NxXD-B>(dDKN>sqI!ZvYF{>S5B zd;t`}zX?klZSxTjRdHjjy3OE9*M6F%2?y-{j{Ox+I_Kag@%y=fgajdilV~p>LZp=h zp#XP;ynGee8Hh(@s$bCRYuS2EUSk9|(dnFS8jLfBV>reaRsTJN=Rc{r18515GK`U{ zpp>C*P_rXKc()y^H&Zv4XF=%0t*UkVAw>{`ssykHNq<;gkrQ@o&WV zm1Vqph&$#H0!^|tM}XzDj9<191mY^@BN)w*{rj?;&1Xp710I9nImKRqZy*<#A|C-! z-Zxfn0F&O+u1wlDBLo0}1pW@h`z>|m2GT@F7*IBxXNa%^$N##Ba!Os7lVR~`o;X4v zAdmRn0IWR|LWt4GJd|7#DC!Y(4>7&bZCdk_TODv|gaFaI0k=4^KhXC%LSfiJhY@<6 zA*W8j_Znn>Mq|J7Q9|CpM-g_;fX|wrrRTmsieQB&Sq$iu@r?+_Jc2%A?;+A9RRV5< zNJsj6zVcQ`lMZA9=sPdxa{U#LeFyE+3)xZ1J;Z#ppTu}W{fu}H0MGtI3TfcuAk^QE z3BDFo-at}h4nHM(9EWm$d-F^``1&2$@G~1+*0Y~-ZXj_T0NL0>EHMViQG|IzLO5znRPfNSU} zwypX-SvIdX1|yuTbq-n{at?aLb8X6J6Dy=1Sg6OkU}JO!cw}dp1`qm&YPRUV*U!S2 zn5DFlC(dx|H)p`HU3@7?{Ox5^)}8>~XFP@Mg;hK zuv zqc?c1CSm~#8w{nMr|mswH^IR2&h9x!0GQHGn9{#tbmdE*lqc~&>uF~nSt5ZIQ8{n5 zukD5apt0qh3EhksZo?5e2kIw(e%obrU$rE7Q5ns-`#*#6$(M4S{TQ$eTd)aTm{L*1 z{D;okx6M`j*wrf^&#D|OdFSw)Bc*e>Q5HYOKFiCRg$)Rd9aeWsqFvT?miS0{XQ_fa z+|rEvF|JQ3@cs)^O5h0p2FmHT*o%`<1LK^2bIRP@9m;}uH2{dzEp3OV1)$9f4p_FB z6QSoEC7xz;$g>EU%5zRTWM;W$dP=AD0f@nswlnbDxB#9XGo=B^KJw$5PB@!|JlB9g zg>%U4JY#JQ1m~RA#gXKket;`3Tkz3<6zIm6C5NS2H|N)Paqp4dG@dEhW}h*pzdl~j zb5w~RQ`OG^m%m9|3~kaZm8U$IHj>@&xihVHc42a!S83JjhhD<1ttysKj=gtlC*g3? zGKR?yq8`**efv$ga5Y1~vkZeLPrNwCApTGnFo4Dz^xX$yUXrcwYr|oPiLJ+E(J+Ok zQ1gT>96Lu&MI5b+2NgoKp@=!ew)*z&EF^YO28*Ta1wKp{QxksQ^kU3#rm*Gf&xc(B zdA-9IQA9F&Zzvzi%=djCBAD-f-Lg~+e_jur9B=6vKdOCfU)9jW5>6+K#2iB!`mj_J z2SWK&q+srQoT^oRk&#@;P?D2Oi_n~8sG*F=Psvk2G>6!$iO>aV8Rcp1O?5w|=lsC* z`eunhyN#fUH9x@b&%aSlcO-B1r%YoM3z2vj6BP-R+KdbXr#qA;=m4v z;0}w6yL2*~gq2JYCZu);a;YdtswXKaK0{IfLUNs?IiV;NGE!HXdvrnza|Jqq9E!r( zn^9Bj*R!|-)~IK(CKlpq&u0J1MFnSR*v&;~r^ICm>Z~Y#K(sRPFb^TEi%Q2L+ew*{C7u*s&(1SVNggKU z!>PJ6blEf+HQ|9AusJHo{w>ReJ)9JQ&p);@9Oy(+mF>VF{h+8#NujGGFaE>h0Astg zhB5;D@WAP(>R~Zsaj5URPn+0bG5nMeS3;tf6^CTMNjY}Pc`f3cm=OVO{D&mT z1ayqJqOx+mENOmO6;w>VLxCesU<5904RQn}5;IO9;3f=Ub2kU&Rf5c^gx45)``Vd2 z2Nl8waw#@WfC3qlPlFT4;SSnc&7j8$`4Zg3+9#QGC+4x91+l6WcGq$MLaClpO8Wu@tfLOxoghP&xWF**oaEx zTf~@it=Vf4ruk@^vk3|TudTdR8%!u07{D*p?^~n6~#kRmlt)x#y%7+_Jsy)JSi#O zl;!s*A4u5M$9um4&Jqo-UA^S2rP_c6jcXud`iqW@-rk&*;(=1}zC&N!h! z7GTyn7c^j{!cR!yO_{T_I1PA_QBv4aQ3zDKz2LMTIDG36O_a-q`4*x^5;$okRjy*Z zDw@*C?E-frSdzfwyq3ks;=AHwm+P+S5r%izBUPyG19Z`2cA|K}%+((v$on;1)@HPMaz@jjJD<-(%dLK%I9;Wy6B(P`Cvds!FV&R1A7Bx(nm4 zTu3T^e{{u#WZ95DZ($}TC$uOpFh+>q#D64Yl9#$R7d9Lu7b>ESS?&tQFYser0&{`? z^;t4SJ5UAXirEXvZO{icu%%>=Us#0>X58^FHXFCp<%*@{`d&5+?4;F(bkG#nj4UWG zCeEyhS@=zGh7xt$au?g_;P)42+%c-L@>WERXxEGaDs8FFafwEaXZdZ8Bl8~+?yPO7)uwTuGlh9s;9u)gdgiBz{u&`^WLWDX zX5&Wzj;fmMNj6PDTTAOB0bqzdgo;UHZn^R2kAMQ3GH zSI*Kl!~k$e20hqw>w5`Ir_pgWibo_!3=&)|6B}p~N#sy8Om>Cq#(L@`PmwAx~8H>~V0SdaouT0*0j|h1MjA1Yj7vQ3Z}MsqrC0tmOv)$pi^< zFmcvg!+972qx>cTO2b5VuYfkHSdteEL$eKuwt<0GnR`6=f|mJuu#PMX@85)qd9poJ z9y&>O_$KmpU6bfI3d7?tr&b^nZhy96UQG zxRm}4WQTzbdAAlV*Zwb3%(S<+{z*BnAOGXcWXVD#Z6gpsK%Lb8@g6i63sYNDhQGE< zpRYi=&;q_k2+Gv!Q%6}Yh;Jm`Rc`T2|P-xNL-Vswhg5c(Fs`({Dos*fZ=A{ zAk2BoU|qE!?D_m#rX79@!Y_R+s@I>1s2oIwd&ECQ9!^=23AkPS7ixL;jRD0GdwYqIEwNRLzP?Bx4wS6szFvvPo zDQ$G&#J!@R2b1l{rUYoW2{AGV!H|i|q~ERDH)uVPUNEi;!hbGr{(xGCf$OpuNkWP=0Zo*)P15Rd>+b9G{rzN?rDa7XEBLiu?aIa1`A&-6 zxt8x@G*4)N^)%UK^w#Zzg5G>0xIl!$Sa@K@BqP(%r2M0{ zXi+lT&51{_oEkvcfV(<$FnDQ z`&^%9ps}RBZU5)jyn;rveb^XgDalKK@b=|{I@S|IdQeXN%34i|&Bd6Y7m;rKwd^}rQ0y=$ z*HkNF(jlze22GNQY1j8HK(AhHj#O&QmU2T_%&l~48E-Tcu7rsM?Kj?#C-EbFmKF78 zVzZ-=iETo8sYrUCcWC^2{5+XS(xt2JFYsY6!)>`B{ylGI6{6LWLJj_k<2tX@*wFXA zh>o}fjRxV(LE{y?Xe+Y;7-w{)?l9SLsKOz)3Su*_tW)<@0kz&clTMj+o0K>CmV}jU z{?i9>+m0o?7KtT;>>}w-akDRc&;qX%S!30q${sXyjCJF z{qH(Af4I(SyR73H3cC#_=dbJ72k2yuocw=noqIf!dmP7yu*z*EmvZkixnCl;SA`SH zb&kZ4yTjPb^>pautdqoHbeuJu2t&3+ona}YL`Q!JzJ71Kt!mUO#^r>gHnl)TB@yuF30tWRG$yT4=nd7Kf>hROfLVPSxPgY=l0CkQmYy@2U>n6(1w0tAC!w9Id_g($Dx_( z1Ux9llSY;8lm8XIt?F5Y$oXTKiM!v+ikK9AH2-VV$Ama)g-8|7n-clHpn7|+SZ0QE z`%s#h?eb%(yTm`7MDK$F)HT%m2B>MZ1~vP5g_@ zv@zGy_lsv)(H(Y`jFVH!19}yVg2CxI{J~1)c6$?KFxa^r+eO|CyNxU?ojA?ze|$ zCt+eU!=$jS!`8crg|@aDWz@j;PwA6mArmOZ^B%)Uyg6I(0{4ibv1e{@7A8J_jIQ;a z)bJkt0Hj#{?yL~aJrXK~M}zcJnf|w3b8A_`MHJ8Rr+b5EQhwBu%ALVZw#_URJ@zYq zRT6O1@(VX#XLD?}*{2&PKZpy?H0F7axx;{roe7?@Ycq{cg$B8AQ2&v7?ngW@kbw1^ zzS}aI%lxj1E=t*?kyu@k4Q?Gd_9 zI#}7vzo@!A@d-C!x5h_2LN`40r~KK^sv+b#lv%2kN)5jD+n7cjb~}%(h-q?>YvP$4 zz^3$?2I?fE!%3Zk@6Hnzi1reN_4l?X#7Ihtm#fP~ex4Y_&x)cQg@k2SdIt`ANFBCR zQQdPH^w$zNVE_{K^@RvNgTZ`kfzcN@?EQ}?Mu*0oTt0Dt0uuK*9{B(^;knT%5k%@+&lWsAH_Z2?UT}GZld#_)mugC&a}Q@LqVF{{IMUd?2Oda1Yn=iurmn%snF|0x0H7dH%-u~;06LZs9k)^olrlFsDE~Jg z;6O%3fYJ{n^}tKN64eikj{aItmLLJh!!xxVxGaNwwo3dG{{BBDdXpb%5b`qHDI23G)55`a_zx85m2 zY{=?(0A2zfd^46C^3@^nUHs(l2q#16Ef^8yV0-pnmDKT8KhdC2W+ELgP@cAwO=Qz7}7%974m; zn=bIfJ!pY2 GdF?+Vi`(1) literal 0 HcmV?d00001 diff --git a/fixtures/documents/sample-requirements.md b/fixtures/documents/sample-requirements.md new file mode 100644 index 0000000..f010a07 --- /dev/null +++ b/fixtures/documents/sample-requirements.md @@ -0,0 +1,47 @@ +# NameCheck Requirements + +The NameCheck service screens submitted names against a watch list. Everything in +this file is invented for testing; it describes no real system. + +## 2. Scope + +NameCheck runs as part of the onboarding flow. It is reached over HTTP and it +publishes an audit event for every decision. + +## 4. Functional Requirements + +### 4.3 Manual review + +REQ-NC-017: a manual review must time out after 30 minutes, and the case must +return to the shared review queue when it does. + +REQ-NC-018: the client retries a failed screening call three times before it +reports an error to the caller. + +- The screening endpoint is POST /name-check. +- The result endpoint is GET /name-check/{caseId}. +- The review timeout is configured by `namecheck.review.timeout.minutes`. +- Decisions are published to the `namecheck.decisions` topic. + +```java +public final class NameCheckService { + public ScreeningResult screen(ScreeningRequest request) { + return watchList.match(request.normalizedName()); + } +} +``` + +### 4.4 Error codes + +| Code | Meaning | Retryable | +| --- | --- | --- | +| NC-100 | The submitted name was empty | no | +| NC-101 | The watch list was unavailable | yes | +| NC-102 | The review timed out | yes | + +> Ticket ABC-1234 tracks the rollout of the NC-102 retry behaviour. + +## 5. Data + +The `screening_case` table stores one row per screening decision. See +`sample-schema.sql` for its definition. diff --git a/fixtures/documents/sample-scanned.pdf b/fixtures/documents/sample-scanned.pdf new file mode 100644 index 0000000000000000000000000000000000000000..41ada6211c4afb14e00872a7c7cdefa6765943ea GIT binary patch literal 467 zcmZWmJ5R$f5Z>=s+>(*TcH*>^sti230HP{v6@wv{5L9AhyHerT<7-ozDwaIlci-dA zPFA=zQqCorh35x}$FF)Ue>5U0yYa2{grXi0i64qFjKwZT|t}g{gUV7hF9yb}& zU!#lNbpho-$epo}V;n{5F(-Ybsx{Ykj~fyBg;Y45tLV2TZ+GrJ4iqOXOAhEl)OpM~ zagf9HDd&N@CVXI~x$lD`^95ou$J2EUOcfSFU|!5nDuIrm2Hv%2_gn8B)gVxV_&-=^ z9gS-|W6_uoc4th_Mwpa(gef`D24~tWxx=44x4(os?r$wWXTPcs#)ZR7naCkY?w*$L E2MN1&;s5{u literal 0 HcmV?d00001 diff --git a/fixtures/documents/sample-schema.json b/fixtures/documents/sample-schema.json new file mode 100644 index 0000000..e057c6e --- /dev/null +++ b/fixtures/documents/sample-schema.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.invalid/schemas/screening-case.json", + "title": "ScreeningCase", + "description": "One NameCheck screening case. Invented for testing.", + "type": "object", + "required": ["caseId", "status"], + "properties": { + "caseId": { + "type": "string", + "pattern": "^NC-[0-9]{6}$", + "description": "The case identifier." + }, + "status": { + "$ref": "#/$defs/CaseStatus" + }, + "submittedName": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "reviewer": { + "$ref": "#/$defs/Reviewer" + }, + "externalProfile": { + "$ref": "https://example.invalid/schemas/reviewer-profile.json" + } + }, + "$defs": { + "CaseStatus": { + "type": "string", + "enum": ["pending", "cleared", "review", "rejected"], + "description": "The lifecycle status of a screening case." + }, + "Reviewer": { + "type": "object", + "required": ["id"], + "properties": { + "id": { "type": "string" }, + "team": { "type": "string", "default": "operations" } + } + } + } +} diff --git a/fixtures/documents/sample-schema.sql b/fixtures/documents/sample-schema.sql new file mode 100644 index 0000000..8d5885d --- /dev/null +++ b/fixtures/documents/sample-schema.sql @@ -0,0 +1,33 @@ +-- NameCheck persistence model. Invented for testing; no real system uses it. + +CREATE TABLE screening_case ( + case_id VARCHAR(32) NOT NULL, + submitted_name VARCHAR(200) NOT NULL, + status VARCHAR(16) NOT NULL DEFAULT 'pending', + reviewer_id VARCHAR(32), + created_at TIMESTAMP NOT NULL, + CONSTRAINT pk_screening_case PRIMARY KEY (case_id), + CONSTRAINT ck_screening_status CHECK (status IN ('pending','cleared','review','rejected')) +); + +CREATE INDEX idx_screening_case_status ON screening_case (status, created_at); + +CREATE TABLE watch_list_entry ( + entry_id VARCHAR(32) NOT NULL PRIMARY KEY, + match_name VARCHAR(200) NOT NULL, + source VARCHAR(64) NOT NULL +); + +CREATE VIEW open_screening_case AS + SELECT case_id, submitted_name, created_at + FROM screening_case + WHERE status IN ('pending', 'review'); + +-- Vendor-specific syntax the general parser is not expected to structure. It +-- must be kept verbatim with its exact line range, not dropped or guessed at. +CREATE OR REPLACE PACKAGE BODY namecheck_admin IS + PROCEDURE purge_cases(p_before IN DATE) IS + BEGIN + DELETE FROM screening_case WHERE created_at < p_before; + END purge_cases; +END namecheck_admin; diff --git a/fixtures/documents/sample-tests.xlsx b/fixtures/documents/sample-tests.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..6ff2a02cf7c442b98b63ef8244fa5cfa864edaab GIT binary patch literal 6161 zcmZ{I1ys~q)cw#fbR#K^(kY$J07FQ(z@xjnK}t%xLuz;k(mhB^cS}nMh)4?3->Cop z=acWfS?m7pnpv~=nsd*dbI#o=3P{LA0000TzGwl^YluEQ0sydz3;+}l<3P#+}aaIw8*KY9%`kHI#)B92~uZ~MjlHLW+;gaoC1S2>ur!+xbs#A z9(y4^L-|I2VRM0Azo0jmE4%_G^vhtbO1H!{H5q&CN?gdFLKiobGiSu|KKNmiOMhqY z(`;vHjxmDE{_`N28LGY8Ykd2W$K>g^@ok5+c1B&TX~ws!1+FX^nFL8<^W>l2Z5VIAD`>{162Mv)3hiU zWe5B$2@wGRO!#i(Vs7Wk$?@Zvli2J4KTFBL_v5)-XydezyyizF$QKA&1-SDBC{duC z_R57~=XlHq-eRHzOVrkq`+`Lc(WPWJcxu*qGffC`4UDg>M-!wgrrD_ZBPJ!EE}F_z z>9L7Ray+nTd}l^d;##B~X)u>lEVjt`Ov*8+QdrSh+FDM57w2$fGLYx|1dAWE1))#T zC-<7#y7gF-pU9`vA;|;{OchWS2U7K&#ZT>4n$p$z={k+!P~o^qJZNi9aBR(=!j#}o|mT&TXL5EQhaDIKPrFIw}N{) zIHsB$w}^%ssk^{_E+~u}OK^C-v1)gk18nI&mMZO8Bdp5Qy{^M2T+K{VwnPcU+xB4; zk0w+QN@GR2i&h^ZR?q!3=y+|0(vH?b`e&}Zssx9K;QCL5=L-LyA(%S4nEzCNVl`a- zc;QBjgSpvB(rBzB1R5kjISs^W-Lv|7a*75YgB8v^0bslwXA%kz-}s(U>Ktj>PKd?mOMWv7$ZEmmI}_Ubrvn}>B0s$GUivX`AauEa9}PGG<+<=vdy zzFyMKmJpiN%3I_=j}T_}`AZA@=*W@a3%+~VasE=gr=yFliKCtE;f_utqc~YD?FACwJ5&G!AsMN8||1vW{*S)ewAe_m)!eb=p3}UIz+7wupmKvyVj}HbuSfh@&Jxw){1}{BZ&xE)iX>aM6gx zh}O+khfnV6`fOW|yA|iaw>`kDww(B%5&NAKE~&x=bA|x`VE>BP)y><^-1SGK%UX+3 zi(Gg<10PD9Y8jDIE?|<}_1S zPF%VOE#*YGIMgd zZZQxu;4uTC#nweS88{@M0 z-7ASOoR>?WymG8G(=7kK9Q9gE53-`;$(RLQfmp5s8>I7G8spM~>+TD1ouA6BIb!1n zwm`yIN;m!-pRSr-_1bG<%OA*J-~ zE!$Dr8F(uk>oE@a5@t9x1%0x)A@@G_YM(0B3a!V@=Fb)?5}*b0S!Dxb^`A}IdM;1Jz3hEE?uFfU^em#g z;$?5yxZ9pJ6Z1`lZP6}?l{9!`&xe4xeoJ!dAwb8;V|I8FDb?`NrF$~@2P)XIl)<6S zSD}~qx4rL?XRRk6r1F7Xa^g{N08e@*6jl5xr5d8vx)LW1;%CPm)vJ)BXZ8nwlMSJL zzf^hiEe$qM#OsseFFb0S9B5sJP_hrp8cVu%-Pz15Nt#oV9(8JE!#KL4*Z(Vm*+GQR zMVQi3+HdUaETNc@XYAktxTh>?Nu!lo^?Yn>GdISC9=3d@|?h36KJaMMkx>T5^LyV9{w#xU0pY8puS4zjx+iX!PLV}TGOjN1x zGzKPuQlS7PIl^1k;)UDnDapfRPKHZJ!Yk`q`Vu#$ksV(0PH-iKdxX`bsLw6RMK(W> zGJ%2$(UPJTOdDzqJEX2;h|7H2h0iK2Zt_5Y+@+o*xWTzJ)L4(8cfw0~vJU@Rx12vo z+_?YCNP_$5Yyu8`qTv(xcSob+@!P4x1tFRH1mk(jgSc;XhV<(0nSEKiT+%-rurwDq4R9?4aLsRl&V5$H&29s_in;-=|TT^k`lv zuj9*&m3Z#a{^bXT(8^3*c6yR{h2UB#hNkQ2!NbQr7&~VAj2J^2Q|3>i2y&$<9Cp>e zK~mIa6JbkQNCc@j-DBUupSA~zr-maX14cutlGCv}R_^ z{u=E?r7GgFnx-+gelHShe1V^03~0w`JI%moVh@rmry|Sqe(%(F#^~8PLk8Vf!vw>KNE6f_e zCteBoxQ6>{c=_j`R-d^1u_%^^zg>8XzLdw!rA7jnxF$n_zT| zLJK>M6NO_9M=YPeb(gzB;NFbc1eJ$Tbuk8P?{Nv4_K4|izEb4(#C+_$Gx3pF5sLcw zCFAp4mbDa7Ti5`SZdU1ES@TlH!-4%-j*XHO7BZp)=8u=2_=0(wUbKngb&ju#tZX>&*f)2{m&!Q)Loi>GG^ZBQ2{XI_|tF)HxX*sw^M=y@u$QACkAd7x~gXSc6zVp2Wkgy9XZ_0&lF;_?OapNTlY z1EJc%r${_B0D$awB3!M^&D~r%e{Mh9xqph1I((oEzl-WEIRkhQv<(X8Vkkp|_#_ec zb*ty9qu3u5#n~fhJ2tsn*G90E9h>&X^HyLOu*mn=A1MZ2$NwXw>fkul@kIlv$*C zphU`yFa*d`hm(El5ihN&OApz&Smq^@*84bj6md1y=JCe78@uSuC94ZtNhheog~dz% zBgDOZ?OpB|@;0S6f$a>p{hs3bFpizw0&~$;Em&zU1o5w$P5vTAvuDXRsMszU2r^t& z>v)gYVXVVDvT-l=y{f2u?bHV|mWS`ocNn$B2Y|Wy`N8vrGO3qldsqx}(n5T$c@{^L z<5$5;`MTfv8gr2M@eK6IK#42j95fcI7D+=NKAa1(C3?wbdEj^@A*}Qeex?6szSA^=ffiQb;x&rz~+Z~UW-ICKhUwW ziC0)1o$!pCtHp`?xO6rHHD205w-W>^_MmeOIcCxW=J3X$yT6zY-$|BoFeviFL6gtH zDN@Ew$ZV3kAVMmo;ZPHw3Hj`6o_L9&*<6a!f;+!NL45lyBGDaD0emE@y4kfk1>+7g z$~3|DSv)uC=9z&ko%j9aZgn2*?|Hk_*3KH>`i}Ute(^-w|Mq(}&&1U&Ebq0=CCs7_ zPjeYpp`}K05&N`N1t`HJ(BXu^;JKR? zt9_8H+QkyFI#G({))}-mD4hmX&2SO5wvfeEXjWCEfN|@+`N-!spJSe|ObR1Y1eUB- zb8a>-ks2<)q2*vH6oG|2d429c?Fz+VNx>Xvig+8TLh9loo{MuM(;e|b{3aSjM3*kW z`B;b8r6;6AMn1;)q&oP4*O;Vk?>UY-HH-IQV5@99O{*oRLY8s(?Ud#;(Xmz(&<1vP z8g+y8<=TJ>c}2GdEc*pymIqFwI@T#4Qc*zvf>LZq1IXk}On!|`IEH^nyTP{k6zur`bJ z(Yn@!0PUA>Z9O{ta9X=#Q<=?hKO_1{@sIDR+Fo5C{h62T9FtZsJTDLbJ1@My@-pG@ zf(z?9g@tB(Tz(k}jBin%I^!y}soxhuK~hah7lPUawuh$L;G*Zb3avgk#@{6p29QXM zUaIuRH5;)Kzro4I!Ti$MmxMZC9T%@h?1T9HSz5NHr>;k3wt!*k#q^?T#zx1i*L(u{ z3)jqmypDC9{^D#3-zkr5@;Af2%UuZH3s+U2euk=1pox>6hP-~XY0VIw+8FE3 zCLxg${~1Q>OxSQDZIz0%+W3)sIJ-_(XhG~j8MHN4k@Rk^_Hrm$%tGihK)=oi`l%xR zmD=?9PUkUopLWwH-DNCs+gWpEK_EnEs1*Xrr1I`9xuHxqS?!TBARm)7)W?53Y2dzI zz7L?)rW9k0db3WrWf4t!QwQ@@dV3_LV^!eR;H4ukMJhqsG5-uELC4!t`emIN84~KX z^6(gIQrJNPbb+R#fPhE@_;1S;Ub=t2g5VkXR|EAv!F_-57aRZxLePLy{KGB054`WU z{RWD`m2lsEyN|wab^S&I;G@9*qW{0~bsv7;=J^emfqTdQZTj5jxo@8Q=HW;A$@AMv zxevX6_x%m6hWF5axe4DVxIg>+COC%sIB wS*Qs9mE~uJz0Y#L0{mvFhr9Uy@>_n^0~G}nxO4Esl*R-M!WF|!{NvsK0OCev-v9sr literal 0 HcmV?d00001 diff --git a/openmind/config.py b/openmind/config.py index d0ae3ef..b4c2b1c 100644 --- a/openmind/config.py +++ b/openmind/config.py @@ -183,6 +183,70 @@ def _first_existing(*candidates: str) -> str: OCR_MAX_UPLOAD_BYTES = 50_000_000 # /ocr rejects larger image uploads (413) instead of buffering them +# --------------------------------------------------------------------------- +# Document ingestion policy (OpenMind v2 Phase 3) +# --------------------------------------------------------------------------- +# Code discovery and DOCUMENT discovery are separate policies on purpose. A +# `.pdf` or `.docx` must never reach walker.read_text (it would decode binary +# noise and index it as source), so no document extension is added to +# INDEX_EXTENSIONS; the document walk has its own extension set, its own size +# limit, and its own parser plane. +CODE_INDEX_EXTENSIONS = INDEX_EXTENSIONS # readable alias; same set, unchanged + +TEXT_DOCUMENT_EXTENSIONS = { + ".md", ".markdown", ".rst", ".adoc", ".asciidoc", ".txt", +} +BINARY_DOCUMENT_EXTENSIONS = {".docx", ".pdf", ".xlsx"} +STRUCTURED_DOCUMENT_EXTENSIONS = {".csv", ".tsv"} + +#: Everything a document parser can claim when a user ATTACHES a file explicitly. +#: Wider than the discovery set: `.html`, `.yaml`, `.json` and `.sql` documents +#: are perfectly parseable on demand. +DOCUMENT_EXTENSIONS = (TEXT_DOCUMENT_EXTENSIONS | BINARY_DOCUMENT_EXTENSIONS + | STRUCTURED_DOCUMENT_EXTENSIONS + | {".html", ".htm", ".yaml", ".yml", ".json", ".sql"}) + +#: What a WORKSPACE walk discovers as a document. The subtraction is +#: load-bearing: a file the code pipeline already owns (.html/.yaml/.json/.sql) +#: must not also be ingested as a document, or one file would become two Assets. +#: Those formats stay reachable through `openmind document add`. +DOCUMENT_DISCOVERY_EXTENSIONS = ( + (TEXT_DOCUMENT_EXTENSIONS | BINARY_DOCUMENT_EXTENSIONS + | STRUCTURED_DOCUMENT_EXTENSIONS) - CODE_INDEX_EXTENSIONS) + + +def _int_env(name: str, default: int) -> int: + """An OPENMIND_* integer override, ignoring anything unparseable or <= 0 — + a typo must not silently disable a safety limit.""" + try: + value = int(os.environ.get(name, "").strip()) + except (TypeError, ValueError): + return default + return value if value > 0 else default + + +# Parser resource limits. Every one is enforced by the parser itself, and hitting +# one produces status='partial' + an exact warning naming the limit — never a +# silent truncation. Documents are untrusted input; these are the blast radius. +DOCUMENT_MAX_BYTES = _int_env("OPENMIND_DOCUMENT_MAX_BYTES", 25_000_000) +PDF_MAX_PAGES = _int_env("OPENMIND_PDF_MAX_PAGES", 2_000) +DOCX_MAX_PARAGRAPHS = _int_env("OPENMIND_DOCX_MAX_PARAGRAPHS", 50_000) +DOCX_MAX_TABLES = _int_env("OPENMIND_DOCX_MAX_TABLES", 2_000) +XLSX_MAX_SHEETS = _int_env("OPENMIND_XLSX_MAX_SHEETS", 100) +XLSX_MAX_ROWS_PER_SHEET = _int_env("OPENMIND_XLSX_MAX_ROWS_PER_SHEET", 20_000) +XLSX_MAX_CELLS = _int_env("OPENMIND_XLSX_MAX_CELLS", 500_000) +CSV_MAX_ROWS = _int_env("OPENMIND_CSV_MAX_ROWS", 50_000) +DOCUMENT_MAX_BLOCKS = _int_env("OPENMIND_DOCUMENT_MAX_BLOCKS", 20_000) +DOCUMENT_MAX_BLOCK_CHARS = _int_env("OPENMIND_DOCUMENT_MAX_BLOCK_CHARS", 20_000) + +# ZIP package safety for the OOXML formats (DOCX/XLSX). A document archive is +# never extracted to a filesystem path; these bound what may be read INTO MEMORY +# from an already-validated archive. +ZIP_MAX_MEMBERS = _int_env("OPENMIND_ZIP_MAX_MEMBERS", 2_000) +ZIP_MAX_TOTAL_BYTES = _int_env("OPENMIND_ZIP_MAX_TOTAL_BYTES", 400_000_000) +ZIP_MAX_MEMBER_BYTES = _int_env("OPENMIND_ZIP_MAX_MEMBER_BYTES", 100_000_000) +ZIP_MAX_RATIO = _int_env("OPENMIND_ZIP_MAX_RATIO", 200) + # Template profiles (declarative framework/architecture lenses; see openmind.templates). # Built-ins ship with the package; users drop their own .yaml/.json into the # data-dir folder — same schema, user files override built-ins by name. diff --git a/openmind/documents/__init__.py b/openmind/documents/__init__.py new file mode 100644 index 0000000..e527f66 --- /dev/null +++ b/openmind/documents/__init__.py @@ -0,0 +1,60 @@ +"""Deterministic enterprise-document ingestion (OpenMind v2 Phase 3). + +Importing this package imports NO parser dependency: the registry stores module +paths and imports a parser only when a probe matches it, and each parser imports +its third-party library inside ``parse()``. That is what keeps the +dependency-free ``.openmind`` artifact export runnable on a machine with none of +the document packages installed. + +No model is involved anywhere in here. Parsing is byte-deterministic, structure +is read from the file's own markup, and nothing is inferred about meaning — +Requirement, Business Rule and Design Decision extraction are Phase 4. + + from openmind.documents import parse_bytes + result = parse_bytes(data, logical_key="documents/spec.md", + filename="spec.md") + result.status, result.title, len(result.blocks) +""" +from __future__ import annotations + +from typing import Any, Dict, Optional + +from .models import (DOCUMENT_SCHEMA_VERSION, DocumentBlock, DocumentLimits, + DocumentMetadata, DocumentParseContext, + DocumentParseWarning, DocumentProbe, ParsedDocument, + UnsupportedContent) +from .probe import probe_bytes +from .registry import (AmbiguousParser, DocumentParser, ParserEntry, + ParserRegistryError, list_parsers, parse, parser_names, + select) + + +def parse_bytes(content: bytes, *, logical_key: str, filename: str = "", + workspace_id: str = "", declared_media_type: str = "", + limits: Optional[DocumentLimits] = None, + parser_options: Optional[Dict[str, Any]] = None + ) -> ParsedDocument: + """Probe, select a parser and parse — the one call most callers need. + + Never raises for a bad document: every outcome is a :class:`ParsedDocument` + with an explicit status (``parsed``, ``partial``, ``needs-ocr``, + ``encrypted``, ``unsupported`` or ``failed``). + """ + context = DocumentParseContext( + logical_key=logical_key, filename=filename or logical_key, + workspace_id=workspace_id, declared_media_type=declared_media_type, + limits=limits or DocumentLimits.from_config(), + parser_options=dict(parser_options or {})) + probe = probe_bytes(content, filename=context.filename, + declared_media_type=declared_media_type) + return parse(content, context, probe=probe) + + +__all__ = [ + "DOCUMENT_SCHEMA_VERSION", "DocumentBlock", "DocumentLimits", + "DocumentMetadata", "DocumentParseContext", "DocumentParseWarning", + "DocumentProbe", "ParsedDocument", "UnsupportedContent", + "AmbiguousParser", "DocumentParser", "ParserEntry", "ParserRegistryError", + "list_parsers", "parse", "parse_bytes", "parser_names", "probe_bytes", + "select", +] diff --git a/openmind/documents/builder.py b/openmind/documents/builder.py new file mode 100644 index 0000000..7873b7c --- /dev/null +++ b/openmind/documents/builder.py @@ -0,0 +1,180 @@ +"""Shared block assembly for every parser. + +Ten parsers each enforcing "cap the block count, cap the block length, downgrade +to partial, record the exact warning, keep the heading stack consistent" would be +ten chances to get one of those wrong, and the wrong ones would be the silent +ones. So it is written once here, and a parser only decides WHAT a block is. + +WHAT THE BUILDER GUARANTEES +--------------------------- +* Ordinals are dense and assigned in emission order. +* ``block_key`` is unique within the document (a deterministic ``#n`` suffix + resolves a collision — never a random id, because the key is hashed into the + Revision's ``structure_hash``). +* A block longer than ``max_block_chars`` is cut at the limit AND recorded as a + truncation warning naming the block, and the document becomes ``partial``. + Truncation is never silent. +* Once ``max_blocks`` is reached, further blocks are refused, the document + becomes ``partial``, and exactly ONE limit warning is emitted (not one per + refused block, which would turn a bounded document into an unbounded warning + list). +* The heading stack yields each block's ``heading_path`` and ``parent_key``, so + a parser never maintains that bookkeeping itself. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple + +from ..domain.types import ContentMode, DocumentBlockType +from .models import DocumentBlock, ParsedDocument + +#: Where a truncated block's text is cut. Kept as its own constant so the marker +#: is testable and identical everywhere. +TRUNCATION_MARKER = "\n[truncated by OpenMind: block exceeded the size limit]" + + +class DocumentBuilder: + """Accumulates blocks into a :class:`ParsedDocument` under the parse limits.""" + + def __init__(self, doc: ParsedDocument, *, max_blocks: int, + max_block_chars: int, root_key: str = "doc") -> None: + self.doc = doc + self.max_blocks = max(1, int(max_blocks)) + self.max_block_chars = max(1, int(max_block_chars)) + self.root_key = root_key + self._keys: Dict[str, int] = {} + self._limit_reported = False + #: (level, block_key, heading text) for each open heading, outermost first. + self._headings: List[Tuple[int, str, str]] = [] + + # -- state -------------------------------------------------------------- + @property + def count(self) -> int: + return len(self.doc.blocks) + + @property + def full(self) -> bool: + return self.count >= self.max_blocks + + @property + def heading_path(self) -> List[str]: + return [text for _, _, text in self._headings] + + @property + def current_parent(self) -> str: + """The innermost open heading's key, or the root.""" + return self._headings[-1][1] if self._headings else self.root_key + + def push_heading(self, level: int, key: str, text: str) -> None: + """Open a heading at *level*, closing any equal or deeper ones first. + + Called AFTER the heading's own block is added, so the heading block's own + ``heading_path`` names its ancestors and not itself. + """ + while self._headings and self._headings[-1][0] >= level: + self._headings.pop() + self._headings.append((int(level), key, text)) + + # -- emission ----------------------------------------------------------- + def add(self, block_type: str, text: str, *, key: str, + locator: Optional[Dict[str, Any]] = None, + content_mode: str = ContentMode.VERBATIM, + metadata: Optional[Dict[str, Any]] = None, + parent_key: Optional[str] = None, + heading_path: Optional[List[str]] = None, + indexable: Optional[bool] = None) -> Optional[DocumentBlock]: + """Append one block, or return None when the block budget is spent. + + ``indexable`` defaults to "content-bearing and not a structural + container", which is the rule the vector projection relies on. + """ + if self.full: + self._report_block_limit() + return None + + text = "" if text is None else str(text) + if len(text) > self.max_block_chars: + keep = max(0, self.max_block_chars - len(TRUNCATION_MARKER)) + text = text[:keep] + TRUNCATION_MARKER + self.doc.mark_partial( + "limit-exceeded", + f"block {key!r} exceeded DOCUMENT_MAX_BLOCK_CHARS " + f"({self.max_block_chars}) and was truncated", + limit="DOCUMENT_MAX_BLOCK_CHARS", block=key, + allowed=self.max_block_chars) + + if indexable is None: + indexable = (block_type not in DocumentBlockType.CONTAINERS + and bool(text.strip())) + + block = DocumentBlock( + block_key=self._unique(key), + block_type=block_type, + ordinal=self.count, + text=text, + parent_key=(self.current_parent if parent_key is None else parent_key), + heading_path=(list(self.heading_path) if heading_path is None + else list(heading_path)), + content_mode=content_mode, + locator=dict(locator or {}), + metadata=dict(metadata or {}), + indexable=bool(indexable)) + self.doc.blocks.append(block) + return block + + def add_root(self, title: str, locator: Dict[str, Any], *, + metadata: Optional[Dict[str, Any]] = None) -> DocumentBlock: + """The single ``document`` root block. Always stored, never indexed: it + is scaffolding, and embedding a title on its own dilutes retrieval.""" + block = DocumentBlock( + block_key=self.root_key, block_type=DocumentBlockType.DOCUMENT, + ordinal=self.count, text=title or "", parent_key="", + heading_path=[], content_mode=ContentMode.DERIVED, + locator=dict(locator), metadata=dict(metadata or {}), + indexable=False) + self.doc.blocks.append(block) + self._keys[self.root_key] = 1 + return block + + # -- internals ---------------------------------------------------------- + def _unique(self, key: str) -> str: + """A collision-free block key. Deterministic: the same input sequence + always produces the same keys, because the suffix counts prior uses of + that exact key rather than a global counter.""" + key = key or "block" + seen = self._keys.get(key, 0) + self._keys[key] = seen + 1 + return key if seen == 0 else f"{key}#{seen}" + + def _report_block_limit(self) -> None: + if self._limit_reported: + return + self._limit_reported = True + self.doc.mark_partial( + "limit-exceeded", + f"document reached DOCUMENT_MAX_BLOCKS ({self.max_blocks}); the " + f"remaining structure was not extracted", + limit="DOCUMENT_MAX_BLOCKS", allowed=self.max_blocks) + + +def slug(text: str, limit: int = 48) -> str: + """A short, deterministic, filesystem-safe fragment of *text*. + + Used inside block keys so a key reads as ``h2-interfaces-namecheck`` rather + than ``h2-7``. Lower-cased, non-alphanumerics collapsed to single hyphens. + """ + out: List[str] = [] + prev_dash = False + for ch in (text or "").lower(): + if ch.isalnum(): + out.append(ch) + prev_dash = False + elif not prev_dash and out: + out.append("-") + prev_dash = True + if len(out) >= limit: + break + return "".join(out).strip("-") + + +__all__ = ["DocumentBuilder", "TRUNCATION_MARKER", "slug"] diff --git a/openmind/documents/csv_parser.py b/openmind/documents/csv_parser.py new file mode 100644 index 0000000..c2778da --- /dev/null +++ b/openmind/documents/csv_parser.py @@ -0,0 +1,236 @@ +"""CSV and TSV, using the standard-library reader. + +WHY THE STDLIB +-------------- +``csv`` implements RFC 4180 quoting, embedded newlines and escape handling +correctly, has no dependencies, and cannot be talked into evaluating anything. +Dialect detection uses ``csv.Sniffer`` on a BOUNDED sample — an unbounded sniff +on a hostile file is a denial-of-service in one line of code. + +MALFORMED INPUT IS REPORTED, NOT REINTERPRETED +---------------------------------------------- +Two things a naive importer does silently, and this one refuses to: + +* **Guessing a dialect it could not detect.** When the sniffer fails, the parser + falls back to the extension's conventional delimiter AND records a + ``dialect-uncertain`` warning, so a mis-split file is visible rather than + mysterious. +* **Reshaping ragged rows.** A row whose field count differs from the header's + is kept as-is, counted, and reported in a ``ragged-rows`` warning. Padding or + truncating it to fit would fabricate data. + +Header detection is deterministic: the first row is the header when every one of +its fields is non-empty and none of them parses as a number. Anything less +certain is treated as data, and the columns are named ``col1..colN``. +""" +from __future__ import annotations + +import csv +import io +import os +from typing import Any, Dict, List, Optional + +from ..domain.types import ContentMode, DocumentBlockType +from .builder import DocumentBuilder +from .models import DocumentParseContext, DocumentProbe, ParsedDocument +from .security import decode_text + +#: Bounded sniff window. Big enough to see a representative sample, small enough +#: that sniffing a hostile 25 MB file is still instant. +_SNIFF_BYTES = 16_384 +_EXTENSIONS = frozenset({".csv", ".tsv"}) +_DEFAULT_DELIMITER = {".csv": ",", ".tsv": "\t"} + +#: Column-name limit for a header row. A "CSV" with 10k columns is a +#: transposed data dump, not a document. +_MAX_COLUMNS = 512 + + +def CLAIMS(probe: DocumentProbe) -> bool: # noqa: N802 - registry protocol + if probe.magic.get("binary") or probe.magic.get("zip") \ + or probe.magic.get("pdf"): + return False + return probe.extension in _EXTENSIONS + + +class CsvParser: + """Row-oriented CSV/TSV extraction with bounded dialect detection.""" + + name = "csv" + version = "1.0" + + def supports(self, probe: DocumentProbe) -> bool: + return CLAIMS(probe) + + def parse(self, content: bytes, + context: DocumentParseContext) -> ParsedDocument: + text, encoding, lossy = decode_text(content) + ext = os.path.splitext(context.logical_key)[1].lower() + doc = ParsedDocument( + parser_name=self.name, parser_version=self.version, + media_type=("text/tab-separated-values" if ext == ".tsv" + else "text/csv"), + title=context.filename or context.logical_key) + if lossy: + doc.add_warning( + "decode-fallback", + "the file is not valid UTF-8; it was decoded with replacement " + "characters, so some values may not be exact", encoding=encoding) + doc.metadata.extra["encoding"] = encoding + + delimiter, sniffed = _detect_delimiter(text, ext, doc) + doc.metadata.extra["delimiter"] = ("\\t" if delimiter == "\t" + else delimiter) + doc.metadata.extra["dialect_detected"] = sniffed + + builder = DocumentBuilder(doc, max_blocks=context.limits.max_blocks, + max_block_chars=context.limits.max_block_chars) + key = context.logical_key + builder.add_root(doc.title, _locator(key, "A1")) + + reader = csv.reader(io.StringIO(text, newline=""), delimiter=delimiter) + try: + rows = _read_bounded(reader, context.limits.csv_max_rows, doc) + except csv.Error as exc: + # A structurally broken file (an unterminated quote spanning the + # whole document) is reported as partial with whatever was read, + # never silently re-parsed with different rules. + doc.mark_partial("malformed-csv", + f"the file could not be read past a CSV error: {exc}") + rows = [] + + if not rows: + doc.coverage["rows"] = 0 + doc.coverage["columns"] = 0 + return doc + + headers, data_rows, header_row_no = _split_header(rows) + doc.metadata.extra["has_header"] = header_row_no == 1 + table_key = "table-0001" + builder.add(DocumentBlockType.TABLE, " | ".join(headers), key=table_key, + locator=_locator(key, _range(1, len(headers))), + content_mode=ContentMode.DERIVED, indexable=False, + metadata={"columns": len(headers), "rows": len(data_rows), + "headers": ", ".join(headers[:_MAX_COLUMNS])}) + + ragged = 0 + for offset, cells in enumerate(data_rows): + row_no = header_row_no + offset + 1 + if len(cells) != len(headers): + ragged += 1 + pairs = [f"{headers[c] if c < len(headers) else f'col{c + 1}'}: {v}" + for c, v in enumerate(cells) if str(v).strip()] + if not pairs: + continue + builder.add( + DocumentBlockType.TABLE_ROW, "; ".join(pairs), + key=f"{table_key}-r{row_no:06d}", + locator=_locator(key, _range(row_no, len(cells)), + row_index=row_no), + # DERIVED: `header: value` is a rendering of the row, not the + # file's literal bytes. The locator still cites the exact row. + content_mode=ContentMode.DERIVED, parent_key=table_key, + metadata={"row": row_no, "cells": len(cells)}) + if builder.full: + break + + if ragged: + doc.add_warning( + "ragged-rows", + f"{ragged} row(s) do not have the header's {len(headers)} " + f"field(s); they were kept exactly as read, not padded or " + f"truncated", + rows=ragged, expected=len(headers)) + doc.coverage["rows"] = len(data_rows) + doc.coverage["columns"] = len(headers) + return doc + + +def _detect_delimiter(text: str, ext: str, doc: ParsedDocument) -> tuple: + """(delimiter, was_sniffed). Falls back to the extension's convention and + says so, rather than guessing silently.""" + sample = text[:_SNIFF_BYTES] + if sample.strip(): + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t|") + return dialect.delimiter, True + except csv.Error: + pass + fallback = _DEFAULT_DELIMITER.get(ext, ",") + doc.add_warning( + "dialect-uncertain", + f"the CSV dialect could not be detected from the first " + f"{_SNIFF_BYTES} bytes; the conventional delimiter for {ext or 'csv'} " + f"({'tab' if fallback == chr(9) else fallback!r}) was used", + delimiter=("\\t" if fallback == "\t" else fallback)) + return fallback, False + + +def _read_bounded(reader: Any, max_rows: int, + doc: ParsedDocument) -> List[List[str]]: + rows: List[List[str]] = [] + for row in reader: + if len(rows) >= max_rows: + doc.mark_partial( + "limit-exceeded", + f"the file has more than CSV_MAX_ROWS ({max_rows}) rows; the " + f"remainder was not read", + limit="CSV_MAX_ROWS", allowed=max_rows) + break + if any(str(c).strip() for c in row): + rows.append([str(c) for c in row]) + return rows + + +def _looks_numeric(value: str) -> bool: + v = (value or "").strip().replace(",", "").replace("%", "") + if not v: + return False + try: + float(v) + return True + except ValueError: + return False + + +def _split_header(rows: List[List[str]]) -> tuple: + """(headers, data_rows, header_row_number). + + The first row is the header only when every field is non-empty and none is + numeric. Otherwise the data starts at row 1 and columns are positional — + inventing header names from a data row would mislabel every citation. + """ + first = rows[0] + is_header = (len(first) > 0 + and all(str(c).strip() for c in first) + and not any(_looks_numeric(c) for c in first)) + if is_header: + return ([str(c).strip() for c in first[:_MAX_COLUMNS]], rows[1:], 1) + width = max(len(r) for r in rows) + return ([f"col{i + 1}" for i in range(min(width, _MAX_COLUMNS))], rows, 0) + + +def _column_letter(index: int) -> str: + """1-based column index -> spreadsheet letter (1 -> A, 27 -> AA).""" + letters = "" + while index > 0: + index, rem = divmod(index - 1, 26) + letters = chr(ord("A") + rem) + letters + return letters or "A" + + +def _range(row_no: int, width: int) -> str: + last = _column_letter(max(1, width)) + return f"A{row_no}:{last}{row_no}" + + +def _locator(document: str, cell_range: str, + row_index: Optional[int] = None) -> Dict[str, Any]: + loc: Dict[str, Any] = {"kind": "spreadsheet-range", "document": document, + "sheet": "", "range": cell_range} + if row_index is not None: + loc["rowIndex"] = row_index + return loc + + +__all__ = ["CsvParser", "CLAIMS"] diff --git a/openmind/documents/docx_parser.py b/openmind/documents/docx_parser.py new file mode 100644 index 0000000..21c20ff --- /dev/null +++ b/openmind/documents/docx_parser.py @@ -0,0 +1,330 @@ +"""DOCX (Office Open XML wordprocessing) via ``python-docx``. + +SAFETY BEFORE PARSING +--------------------- +A DOCX is a ZIP of XML. Both halves are hostile-input surfaces, so both are +bounded before ``python-docx`` sees a single byte: + +1. :func:`~openmind.documents.security.inspect_zip` validates the package — + member count, per-member size, total expansion, compression ratio and path + traversal. Nothing is ever extracted to a filesystem path; the archive is + read from memory. +2. :func:`~openmind.documents.security.harden_xml` applies + ``defusedxml.defuse_stdlib()`` so the stdlib XML parsers ``python-docx`` uses + refuse external entities, external DTDs and entity-expansion bombs. If + ``defusedxml`` is not installed this parser declines the document rather than + parsing it unsafely — an unsafe parse is never the fallback. + +Macros are never executed, external links are never followed, and embedded +images are recorded as unsupported content rather than silently dropped. + +STRUCTURE +--------- +Heading level comes from the paragraph's STYLE (``Heading 1``…``Heading 9``, +``Title``), which is what the author actually asserted — never from font size or +from prose that looks like a heading. Body order is recovered by walking the +document body's XML children, so a table between two paragraphs stays between +them instead of being appended after every paragraph, which is what iterating +``document.paragraphs`` then ``document.tables`` would produce. +""" +from __future__ import annotations + +import io +import re +from typing import Any, Dict, Iterator, List, Optional, Tuple + +from ..domain.types import ContentMode, DocumentBlockType, DocumentParseStatus +from .builder import DocumentBuilder, slug +from .models import (DocumentParseContext, DocumentProbe, ParsedDocument, + dependency_unavailable) +from .probe import _DOCX_MARKER +from .security import DocumentSecurityError, harden_xml, inspect_zip + +_MEDIA = ("application/vnd.openxmlformats-officedocument" + ".wordprocessingml.document") + +_HEADING_STYLE_RE = re.compile(r"^heading\s*(\d)$", re.IGNORECASE) +_CODE_STYLE_RE = re.compile(r"(code|source|listing|monospace|preformatted)", + re.IGNORECASE) +_LIST_STYLE_RE = re.compile(r"(list|bullet)", re.IGNORECASE) + +#: OOXML namespace for wordprocessing elements, used for the body walk. +_W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" + + +def CLAIMS(probe: DocumentProbe) -> bool: # noqa: N802 - registry protocol + """A DOCX is claimed on PACKAGE STRUCTURE, never on the suffix. + + A renamed ZIP with a ``.docx`` extension does not contain + ``word/document.xml`` and is therefore not claimed — it falls through to be + reported as unsupported, which is the honest answer. + """ + return probe.is_zip_package and probe.has_zip_member(_DOCX_MARKER) + + +class DocxParser: + """Paragraphs, heading hierarchy, lists and tables from a DOCX package.""" + + name = "docx" + version = "1.0" + + def supports(self, probe: DocumentProbe) -> bool: + return CLAIMS(probe) + + def parse(self, content: bytes, + context: DocumentParseContext) -> ParsedDocument: + limits = context.limits + doc = ParsedDocument(parser_name=self.name, parser_version=self.version, + media_type=_MEDIA, + title=context.filename or context.logical_key) + + if not harden_xml(): + return dependency_unavailable(context.filename, self.name, + "defusedxml") + try: + import docx # python-docx + except ImportError: + return dependency_unavailable(context.filename, self.name, + "python-docx") + + try: + package = inspect_zip( + content, max_members=limits.zip_max_members, + max_total_bytes=limits.zip_max_total_bytes, + max_member_bytes=limits.zip_max_member_bytes, + max_ratio=limits.zip_max_ratio) + except DocumentSecurityError as exc: + doc.status = DocumentParseStatus.UNSUPPORTED + doc.reason = exc.code + doc.add_warning(exc.code, exc.message, **exc.detail) + return doc + + images = sum(1 for m in package["members"] if m.startswith("word/media/")) + if images: + doc.note_unsupported( + "embedded-image", images, + "embedded images are recorded but not extracted; OCR is not " + "performed in this phase") + macros = sum(1 for m in package["members"] + if m.startswith("word/vbaProject")) + if macros: + doc.note_unsupported("macro", macros, + "macros are never read or executed") + + document = docx.Document(io.BytesIO(content)) + _read_core_properties(document, doc) + + builder = DocumentBuilder(doc, max_blocks=limits.max_blocks, + max_block_chars=limits.max_block_chars) + key = context.logical_key + builder.add_root(doc.title, {"kind": "docx-paragraph", "document": key, + "paragraphIndex": 0}) + _walk_body(document, builder, key, doc, limits) + + for block in doc.blocks: + if (block.block_type == DocumentBlockType.HEADING + and block.metadata.get("level", 9) <= 1): + doc.title = block.text.strip() or doc.title + doc.blocks[0].text = doc.title + break + return doc + + +def _read_core_properties(document: Any, doc: ParsedDocument) -> None: + """Copy the OOXML core properties. Never raises — a document with a damaged + or absent core-properties part is still perfectly parseable.""" + try: + props = document.core_properties + except Exception: + return + def _text(value: Any) -> str: + if value is None: + return "" + return value.isoformat() if hasattr(value, "isoformat") else str(value) + + doc.metadata.author = _text(getattr(props, "author", "")) + doc.metadata.created = _text(getattr(props, "created", None)) + doc.metadata.modified = _text(getattr(props, "modified", None)) + doc.metadata.language = _text(getattr(props, "language", "")) + # `revision` is an explicit, documented DOCX field — the ONLY place a + # version label may come from here. Nothing is inferred from prose. + revision = getattr(props, "revision", None) + if revision: + doc.metadata.version_label = str(revision) + title = _text(getattr(props, "title", "")) + if title.strip(): + doc.title = title.strip() + subject = _text(getattr(props, "subject", "")) + if subject: + doc.metadata.extra["subject"] = subject + category = _text(getattr(props, "category", "")) + if category: + doc.metadata.extra["category"] = category + + +def _iter_body(document: Any) -> Iterator[Tuple[str, Any]]: + """Yield ``("paragraph"|"table", element)`` in true document order. + + ``document.paragraphs`` and ``document.tables`` are two separate flat lists, + so using them loses the interleaving — every table would land after every + paragraph and a table's caption would end up nowhere near it. Walking the + body's XML children preserves the order the author wrote. + """ + from docx.table import Table + from docx.text.paragraph import Paragraph + + body = document.element.body + for child in body.iterchildren(): + if child.tag == f"{_W}p": + yield "paragraph", Paragraph(child, document) + elif child.tag == f"{_W}tbl": + yield "table", Table(child, document) + + +def _style_name(paragraph: Any) -> str: + try: + return (paragraph.style.name or "").strip() + except Exception: + return "" + + +def _heading_level(style: str) -> Optional[int]: + if style.lower() == "title": + return 1 + match = _HEADING_STYLE_RE.match(style) + return int(match.group(1)) if match else None + + +def _column_letter(index: int) -> str: + letters = "" + while index > 0: + index, rem = divmod(index - 1, 26) + letters = chr(ord("A") + rem) + letters + return letters or "A" + + +def _walk_body(document: Any, builder: DocumentBuilder, key: str, + doc: ParsedDocument, limits: Any) -> None: + paragraph_index = 0 + table_index = 0 + truncated_paragraphs = False + truncated_tables = False + + for kind, element in _iter_body(document): + if builder.full: + break + if kind == "paragraph": + paragraph_index += 1 + if paragraph_index > limits.docx_max_paragraphs: + if not truncated_paragraphs: + truncated_paragraphs = True + doc.mark_partial( + "limit-exceeded", + f"the document has more than DOCX_MAX_PARAGRAPHS " + f"({limits.docx_max_paragraphs}) paragraphs; the " + f"remainder was not extracted", + limit="DOCX_MAX_PARAGRAPHS", + allowed=limits.docx_max_paragraphs) + continue + _emit_paragraph(builder, key, element, paragraph_index) + else: + table_index += 1 + if table_index > limits.docx_max_tables: + if not truncated_tables: + truncated_tables = True + doc.mark_partial( + "limit-exceeded", + f"the document has more than DOCX_MAX_TABLES " + f"({limits.docx_max_tables}) tables; the remainder was " + f"not extracted", + limit="DOCX_MAX_TABLES", allowed=limits.docx_max_tables) + continue + _emit_table(builder, key, element, table_index) + + doc.coverage["paragraphs"] = paragraph_index + doc.coverage["tables"] = table_index + + +def _emit_paragraph(builder: DocumentBuilder, key: str, paragraph: Any, + index: int) -> None: + text = (paragraph.text or "").strip() + if not text: + return + style = _style_name(paragraph) + locator: Dict[str, Any] = {"kind": "docx-paragraph", "document": key, + "paragraphIndex": index} + level = _heading_level(style) + if level is not None: + locator["headingPath"] = list(builder.heading_path) + block = builder.add( + DocumentBlockType.HEADING, text, + key=f"h{level}-{slug(text) or index}", locator=locator, + metadata={"level": level, "style": style}) + if block is not None: + builder.push_heading(level, block.block_key, text) + return + + locator["headingPath"] = list(builder.heading_path) + if _CODE_STYLE_RE.search(style): + block_type = DocumentBlockType.CODE_BLOCK + elif _LIST_STYLE_RE.search(style): + block_type = DocumentBlockType.LIST_ITEM + else: + block_type = DocumentBlockType.PARAGRAPH + builder.add(block_type, text, key=f"p-{index:06d}", locator=locator, + metadata={"style": style} if style else {}) + + +def _emit_table(builder: DocumentBuilder, key: str, table: Any, + table_index: int) -> None: + rows = list(table.rows) + header: List[str] = [] + if rows: + header = [(_cell_text(c)) for c in rows[0].cells] + + table_key = f"table-{table_index:04d}" + builder.add( + DocumentBlockType.TABLE, " | ".join(header), key=table_key, + locator={"kind": "docx-table-row", "document": key, + "tableIndex": table_index, "rowIndex": 0, + "cellRange": f"A1:{_column_letter(max(1, len(header)))}1"}, + content_mode=ContentMode.DERIVED, indexable=False, + metadata={"tableIndex": table_index, "rows": len(rows), + "columns": len(header), "headers": ", ".join(header)}) + + for row_no, row in enumerate(rows, start=1): + if builder.full: + break + cells = [_cell_text(c) for c in row.cells] + if not any(c.strip() for c in cells): + continue + # The header row is emitted too (it is real content someone may search + # for), but labelled positionally — pairing it with itself would produce + # the useless "Requirement: Requirement". + names = ([_column_letter(i + 1) for i in range(len(cells))] + if row_no == 1 else header) + pairs = [f"{names[i] if i < len(names) and names[i] else f'col{i + 1}'}" + f": {value}" + for i, value in enumerate(cells) if value.strip()] + builder.add( + DocumentBlockType.TABLE_ROW, "; ".join(pairs), + key=f"{table_key}-r{row_no:04d}", + locator={"kind": "docx-table-row", "document": key, + "tableIndex": table_index, "rowIndex": row_no, + "cellRange": f"A{row_no}:" + f"{_column_letter(max(1, len(cells)))}{row_no}"}, + # DERIVED: `header: value` is a rendering of the row, not the + # document's literal text. The locator still cites the exact row. + content_mode=ContentMode.DERIVED, parent_key=table_key, + metadata={"row": row_no, "cells": len(cells), + "header_row": row_no == 1}) + + +def _cell_text(cell: Any) -> str: + try: + return " ".join((cell.text or "").split()) + except Exception: + return "" + + +__all__ = ["DocxParser", "CLAIMS"] diff --git a/openmind/documents/html_parser.py b/openmind/documents/html_parser.py new file mode 100644 index 0000000..d3fe425 --- /dev/null +++ b/openmind/documents/html_parser.py @@ -0,0 +1,346 @@ +"""HTML structure extraction, using the standard library only. + +WHY THE STDLIB PARSER +--------------------- +``html.parser.HTMLParser`` is a tolerant, streaming, pure-Python tokenizer with +no network stack, no CSS engine and no JavaScript. That is exactly the right +capability envelope for untrusted input: there is nothing in it that *can* fetch +a linked resource or run a script, so "do not execute JavaScript, do not fetch +linked resources" is guaranteed by construction rather than by discipline. +Adding a heavier parser would buy fidelity we do not need and an attack surface +we would then have to bound. + +WHAT IS DROPPED, ON PURPOSE +--------------------------- +`` + +

real content

+ + + +""" +html = parse_bytes(hostile, logical_key="documents/h.html", filename="h.html") +body = " ".join(b.text for b in html.blocks) +check("html: script content is not indexed", "evil.invalid" not in body) +check("html: the cookie-stealing script text is absent", + "document.cookie" not in body) +check("html: style content is not indexed", "background" not in body) +check("html: hidden content is not indexed", "hidden secret" not in body) +check("html: noscript content is not indexed", "noscript secret" not in body) +check("html: real content IS indexed", "real content" in body) +check("html: an image is recorded, not fetched", + any(u.kind == "image" for u in html.unsupported_content)) + +# --------------------------------------------------------------------------- +# 7. Remote references are never fetched +# --------------------------------------------------------------------------- +remote_schema = b"""{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "R", "type": "object", + "properties": {"a": {"$ref": "https://example.invalid/remote.json"}}, + "$defs": {"B": {"type": "string"}} +}""" +schema = parse_bytes(remote_schema, logical_key="documents/r.json", + filename="r.json") +check("json-schema: a remote $ref is recorded as unsupported", + any(u.kind == "external-reference" for u in schema.unsupported_content)) +check("json-schema: the remote $ref is left unresolved", + not any("resolved type" in b.text and "example.invalid" in b.text + for b in schema.blocks)) +check("json-schema: the parse still succeeds without the remote target", + schema.status == DocumentParseStatus.PARSED) + +remote_api = b"""openapi: 3.0.0 +info: {title: R, version: '1'} +paths: + /x: + get: + responses: + '200': + content: + application/json: + schema: + $ref: 'https://example.invalid/schemas/x.json' +""" +api = parse_bytes(remote_api, logical_key="documents/r.yaml", filename="r.yaml") +check("openapi: a remote $ref is recorded as unsupported", + any(u.kind == "external-reference" for u in api.unsupported_content)) + +# YAML must never construct arbitrary Python objects. +evil_yaml = (b"openapi: 3.0.0\ninfo: {title: X, version: '1'}\n" + b"paths: !!python/object/apply:os.system ['echo pwned']\n") +yaml_doc = parse_bytes(evil_yaml, logical_key="documents/e.yaml", + filename="e.yaml") +check("openapi: a YAML python/object tag is refused (safe_load only)", + yaml_doc.status in (DocumentParseStatus.UNSUPPORTED, + DocumentParseStatus.FAILED)) + +# --------------------------------------------------------------------------- +# 8. Oversized inputs +# --------------------------------------------------------------------------- +big_pdf = parse_bytes((FIXTURES / "sample-design.pdf").read_bytes(), + logical_key="documents/b.pdf", filename="b.pdf", + limits=DocumentLimits(max_bytes=200)) +check("an oversized PDF is refused before parsing", + big_pdf.status == DocumentParseStatus.UNSUPPORTED) +check("the oversize refusal names DOCUMENT_MAX_BYTES", + any(w.detail.get("limit") == "DOCUMENT_MAX_BYTES" + for w in big_pdf.warnings)) + +huge_xlsx = parse_bytes((FIXTURES / "sample-tests.xlsx").read_bytes(), + logical_key="documents/h.xlsx", filename="h.xlsx", + limits=DocumentLimits(xlsx_max_cells=3)) +check("an oversized workbook becomes partial, per policy", + huge_xlsx.status == DocumentParseStatus.PARTIAL) +check("the workbook cap is named in the warning", + any(w.detail.get("limit") in ("XLSX_MAX_CELLS", "XLSX_MAX_ROWS_PER_SHEET") + for w in huge_xlsx.warnings)) +check("content read before the cap is KEPT, not discarded", + any(b.text.strip() for b in huge_xlsx.blocks)) + +blocky = parse_bytes(b"\n\n".join(b"paragraph %d" % i for i in range(50)), + logical_key="documents/many.txt", filename="many.txt", + limits=DocumentLimits(max_blocks=5)) +check("a block-count cap produces partial", + blocky.status == DocumentParseStatus.PARTIAL) +check("the block cap emits exactly ONE warning, not one per refused block", + sum(1 for w in blocky.warnings + if w.detail.get("limit") == "DOCUMENT_MAX_BLOCKS") == 1) +check("the block cap is actually enforced", len(blocky.blocks) <= 5) + +long_block = parse_bytes(b"# T\n\n" + b"x" * 5000, + logical_key="documents/l.md", filename="l.md", + limits=DocumentLimits(max_block_chars=200)) +check("an oversized block is truncated to the limit", + all(len(b.text) <= 200 for b in long_block.blocks)) +check("block truncation is REPORTED, never silent", + long_block.status == DocumentParseStatus.PARTIAL + and any(w.detail.get("limit") == "DOCUMENT_MAX_BLOCK_CHARS" + for w in long_block.warnings)) +check("a truncated block says so in its text", + any("truncated by OpenMind" in b.text for b in long_block.blocks)) + +# --------------------------------------------------------------------------- +# 9. Decoding never fails, and a lossy decode is disclosed +# --------------------------------------------------------------------------- +text, encoding, lossy = security.decode_text("héllo wörld".encode("utf-8")) +check("decode: UTF-8 round-trips", text == "héllo wörld" and not lossy) +text, encoding, lossy = security.decode_text(b"\xff\xfeh\x00i\x00") +check("decode: a UTF-16 BOM is honoured", text == "hi") +text, encoding, lossy = security.decode_text(b"caf\xe9 latte") +check("decode: undecodable UTF-8 falls back rather than raising", + text.startswith("caf") and text.endswith(" latte")) +check("decode: a single-byte fallback is NOT mis-decoded as UTF-16", + encoding in ("cp1252", "latin-1")) +mixed = parse_bytes(b"# T\n\ncaf\xe9 body\n", logical_key="documents/m.md", + filename="m.md") +check("decode: a document with a non-UTF-8 encoding still parses", + mixed.status in (DocumentParseStatus.PARSED, DocumentParseStatus.PARTIAL)) +check("decode: the encoding actually used is recorded", + bool(mixed.metadata.extra.get("encoding"))) + +# --------------------------------------------------------------------------- +# 10. A malformed document never escalates into a crash +# --------------------------------------------------------------------------- +for name, payload in (("truncated docx", (FIXTURES / "sample-requirements.docx") + .read_bytes()[:400]), + ("truncated pdf", (FIXTURES / "sample-design.pdf") + .read_bytes()[:120]), + ("truncated xlsx", (FIXTURES / "sample-tests.xlsx") + .read_bytes()[:300]), + ("empty file", b""), + ("random bytes", os.urandom(4096))): + result = parse_bytes(payload, logical_key="documents/x", filename="x") + check(f"a {name} returns a status instead of raising", + result.status in DocumentParseStatus.VALUES) + check(f"a {name} never claims to be fully parsed with fabricated content", + result.status != DocumentParseStatus.PARSED + or not [b for b in result.blocks if b.indexable]) + +# --------------------------------------------------------------------------- +bad = [d for d, ok in _results if not ok] +print(f"\n{len(_results) - len(bad)} passed, {len(bad)} failed") +sys.exit(1 if bad else 0) From ce351fb3d6634e352ee19f85f8b15252882c6652 Mon Sep 17 00:00:00 2001 From: HelloThisWorld Date: Tue, 21 Jul 2026 00:53:10 +0800 Subject: [PATCH 6/9] Phase 3 (docs): README, CLI guide, migration guide - README: an "Enterprise Documents (v2 Phase 3)" section, the document row in "What It Builds", the six additive MCP tools with the reason there is no document-write tool, document commands in the quick start, the new test scripts, and a roadmap that now lists OCR and Requirement extraction as explicitly NOT implemented rather than leaving it implied. - docs/cli.md: the `document` and `knowledge` command groups, the five import decisions in a table, how Evidence survives a parser upgrade, why related results are only candidates, why OCR is absent, why Requirement extraction is Phase 4, why .openmind stays at 1.1.0, and the full table of parser resource limits with their environment variables. - docs/database-migrations.md: v0004 in the migration table, why segments.content_blob_hash and jobs.payload_json exist, why v0004 declares upgrade() instead of STATEMENTS, and the Phase 2 -> Phase 3 upgrade path. - docs/v2/phase-2-asset-model.md: a status note on its deferred list, so a reader is not told document parsing is missing when Phase 3 has shipped it. The original text is left intact as a record of what Phase 2 itself did. Verified out of band: - with docx/pypdf/openpyxl/defusedxml all blocked, the artifact export still runs, the registry still loads, Markdown/HTML/CSV still parse, and DOCX/PDF/ XLSX report dependency_unavailable for the RIGHT parser instead of falling through to one that would mangle them; - a database populated by the Phase 2 build and opened by this one migrates 3 -> 4 with zero differences across projects, jobs, file index, assets, revisions, segments, evidence, Ask history, kv and project meta; legacy code segments keep content_blob_hash='' (not backfilled), and the upgraded database then accepts and searches a document. --- README.md | 137 +++++++++++++++++++++++++++++---- docs/cli.md | 126 ++++++++++++++++++++++++++++++ docs/database-migrations.md | 43 ++++++++++- docs/v2/phase-2-asset-model.md | 9 +++ 4 files changed, 295 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 26f7fd7..04eddca 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ Point Open Mind at a local repository and it builds persisted artifacts: |---|---| | **Source-traceable knowledge index** | Stores repo-relative paths, content hashes, source locations, file metadata, and search chunks. | | **Canonical Asset model** (v2 Phase 2) | Every indexed file is an **Asset**; every observed version an immutable **Revision**; each revision divided into deterministic **Segments**, each with source-locatable **Evidence**. Historical content is snapshotted in an immutable SHA-256 content store, so evidence for an old revision stays readable after the file changes. Unchanged re-ingestion creates no revision and never re-embeds; removed files are marked removed without erasing history. | +| **Enterprise documents** (v2 Phase 3) | Markdown, text/RST/AsciiDoc, HTML, CSV, DOCX, text-based PDF, XLSX, OpenAPI, JSON Schema and SQL DDL become Assets with the same Revision/Segment/Evidence discipline. Parsing is deterministic and model-free; each block's exact text is snapshotted so a citation survives a parser upgrade. Documents can be appended at any time from anywhere on the machine, live in their own vector collection (the code `search` contract is untouched), and are never merged just because two filenames match. Image-only PDFs are *detected* (`needs-ocr`) — OCR is not performed and never claimed. | | **Verbatim glossary** | Extracts terms/acronyms from definition tables, definition lines, acronym expansions, and code comments; definitions are copied verbatim and carry `source_file`, `line_number`, and `content_hash`. The interactive learn path scans code/config sources; the `.openmind` export walk additionally scans README/docs/GLOSSARY files (primary definition sources). | | **Structure and graph map** | Builds a module tree, per-file definition index, import/dependency graph, entry points, and a name-based call/usage graph. Ambiguous call edges are flagged instead of guessed. | | **Exact-token + hybrid search** | Bare identifiers use token-boundary matching; natural-language queries use vector + lexical retrieval. | @@ -137,11 +138,69 @@ Workspace the project (id p_*; the REST API still says /pr and read-only MCP tools (`list_assets`, `get_asset`, `get_asset_revisions`, `get_evidence`) usable straight from Claude Code. -Deliberately **not** in this phase (still later v2 work): PDF/DOCX/XLSX parsing, -requirement and business-rule extraction, Claim/Relation tables, Knowledge-Graph -edges and requirement-to-code traceability. Nothing here claims document -knowledge or traceability exists yet. Full design: -[docs/v2/phase-2-asset-model.md](docs/v2/phase-2-asset-model.md). +Full design: [docs/v2/phase-2-asset-model.md](docs/v2/phase-2-asset-model.md). + +--- + +## Enterprise Documents (v2 Phase 3) + +Documents are first-class Assets. A requirements DOCX, a design PDF, a test-case +spreadsheet or an OpenAPI description gets the same treatment a source file +does: immutable Revisions, structural Segments, and Evidence you can cite. + +**Supported formats.** Markdown · plain text / RST / AsciiDoc · HTML · CSV/TSV · +DOCX · text-based PDF · XLSX · OpenAPI (JSON/YAML) · JSON Schema · SQL DDL. +Parsing is deterministic and model-free — no LLM reads your documents. + +```bash +# append a document from anywhere on this machine +python -m openmind.cli document add \ + --workspace p_... --path ./Requirements_v3.docx --wait --json + +# find an exact requirement id, with a citable evidence id per hit +python -m openmind.cli document search --workspace p_... --query REQ-NC-017 --json + +# code and documents together, reported separately +python -m openmind.cli knowledge search --workspace p_... --query "NameCheck timeout" --json +``` + +- **Append at any time, from anywhere.** A workspace can start with no + documents. A document does not have to live under a registered source root — + and its absolute origin path never enters the portable database. The import + job carries a content hash and a filename, nothing else. +- **Two documents are never merged because their filenames match.** Re-adding + identical bytes is a `duplicate` (no job, no revision, no vector duplicate). + A filename collision with *different* content is a `possible_revision`: it + writes **nothing** and hands back the exact commands that resolve it. +- **Structure is preserved, and so is what was dropped.** Heading hierarchy, + tables, spreadsheet ranges, page numbers and JSON Pointers all become + locators. An embedded image, a macro, a hidden sheet or an unparseable vendor + SQL statement is *recorded* as unsupported content with a count — never + silently ignored. +- **Nothing is silently truncated.** Every resource limit produces a `partial` + parse plus a warning naming the exact limit and the observed value. +- **Evidence survives the source.** Each block's exact text is snapshotted as + its own content-addressed blob, so a historical citation is recovered + **without re-running a parser** — a newer parser version cannot rewrite + history. An attachment's current-source status is `not-applicable`, not a + false `missing`. +- **Exact identifiers beat similar-looking text.** A search for `REQ-NC-017` + returns blocks that contain it, never `REQ-NC-0170` and never a paragraph that + merely reads like it. Documents live in their own vector collection, so the + existing code `search` behaves exactly as before. +- **Related results are candidates, not relationships.** After an import, + OpenMind reports *observed mentions* — of files, symbols, requirement ids, + API paths, config keys, database objects and glossary terms — each labelled + `status: "candidate"` with a confidence. Nothing is persisted, and no result + claims a document *implements*, *refines*, *verifies* or *contradicts* + anything. + +Deliberately **not** in this phase: Requirement, Business Rule and Design +Decision extraction; Claim/Relation tables; Knowledge-Graph edges; +requirement-to-code traceability; and **OCR** — an image-only PDF is *detected* +and reported `needs-ocr`, with no text invented for it and no claim that OCR +ran. Full design: +[docs/v2/phase-3-document-ingestion.md](docs/v2/phase-3-document-ingestion.md). --- @@ -338,6 +397,11 @@ python -m openmind.cli asset list --workspace --type source-code --json python -m openmind.cli asset revisions --workspace --asset --json python -m openmind.cli asset evidence --workspace --evidence --json +# append an enterprise document (DOCX / PDF / XLSX / Markdown / OpenAPI / ...) +python -m openmind.cli document add --workspace --path ./Requirements.docx --wait +python -m openmind.cli document search --workspace --query REQ-NC-017 --json +python -m openmind.cli knowledge search --workspace --query "review timeout" --json + # expose the knowledge layer to an editor or agent over MCP python -m openmind.cli mcp serve @@ -433,6 +497,27 @@ Implemented MCP tools: | `propose_fix` | Preview a literal find/replace as a unified diff. | | `apply_fix` | Apply the literal replacement only if the test suite stays green. | +Additive read-only tools — the nine above are a frozen contract, and new +capabilities are added *beside* them, never in place of one: + +| Tool | Purpose | +|---|---| +| `list_assets` | A workspace's canonical Assets (bounded). | +| `get_asset` | One Asset plus its current-revision summary. | +| `get_asset_revisions` | An Asset's revision history, newest first. | +| `get_evidence` | One citation with bounded content recovered from the immutable snapshot. | +| `list_documents` | A workspace's document Assets, filterable by parse status or parser. | +| `get_document` | One document with its full parse summary, warnings and unread content. | +| `get_document_outline` | A bounded structural outline of a document revision. | +| `search_documents` | Document search; exact identifiers outrank similar text. | +| `search_knowledge` | Code and documents, returned as separate sections. | +| `find_document_related_candidates` | Observed mentions, labelled `candidate`, never confirmed relations. | + +There is deliberately **no document-write MCP tool**: importing reads a local +file, and exposing that over MCP would let a client make the server read a path +it chose. Claude Code drives `openmind document add` through its shell instead, +where the command is visible. + --- ## Using Open Mind With MCP-Compatible Agents @@ -611,10 +696,16 @@ openmind/ domain/ typed application errors + the types crossing service calls ports/ the three narrow boundaries services depend on services/ use-case orchestration shared by CLI, MCP and FastAPI - (workspace, ingest, job, asset, export, health + the container) + (workspace, ingest, job, asset, document, export, health) content_store.py immutable SHA-256 content-addressed blob store (revision snapshots) segmentation.py deterministic Segment + Evidence drafts (shares boundaries with rag) - migrations/ versioned, checksummed SQLite schema migrations (v0003 = Asset model) + documents/ the document plane: parser SPI + registry, ten deterministic + parsers, safety limits, import planning, commit pipeline and + candidate association. Imports no parser dependency itself. + document_rag.py document retrieval (vector + exact-token + RRF) and the + combined code/document knowledge search + migrations/ versioned, checksummed SQLite schema migrations + (v0003 = Asset model, v0004 = document ingestion) walker.py selection-aware walk, .gitignore handling, hashing detect.py manifest/language detection and stack cues langspec.py declarative language registry @@ -746,6 +837,15 @@ python tests/verify_runtime.py # runtime bootstrap idempotency, worker op python tests/verify_services.py # application services and their typed errors python tests/verify_cli.py # CLI contract: JSON output, exit codes, every command python tests/verify_adapters.py # REST route, MCP tool and skill-bridge compatibility +python tests/verify_content_store.py # immutable blob store: atomic write, reuse, corruption +python tests/verify_asset_model.py # Asset/Revision/Segment/Evidence lifecycle +python tests/verify_document_registry.py # parser SPI: probing, single selection, ambiguity +python tests/verify_document_parsers.py # every format's mandatory cases + determinism +python tests/verify_document_security.py # ZIP bombs, XML entities, no formula/script execution +python tests/verify_document_ingest.py # append, duplicate, collision, removal, evidence +python tests/verify_document_search.py # exact-identifier ranking, candidates never confirmed +python tests/verify_document_cli.py # document CLI: JSON, exit codes, bounds +python tests/verify_document_adapters.py # additive REST/MCP + the compatibility gate ``` The remaining `tests/verify_*.py` files are regression suites (Ask flows, model @@ -764,22 +864,27 @@ against the neutral fixture repos in `fixtures/`. The following are not claimed as complete in the current build: -**v2 enterprise knowledge layer.** Two foundation phases have shipped: +**v2 enterprise knowledge layer.** Three phases have shipped: Phase 1 is the tool-first runtime -([docs/v2/phase-1-core-foundation.md](docs/v2/phase-1-core-foundation.md)), and +([docs/v2/phase-1-core-foundation.md](docs/v2/phase-1-core-foundation.md)), Phase 2 is the canonical **Asset / Revision / Segment / Evidence** model plus the immutable content store -([docs/v2/phase-2-asset-model.md](docs/v2/phase-2-asset-model.md)). The following -later-phase items are **not** implemented; the foundation creates extension -points for them rather than building them: - +([docs/v2/phase-2-asset-model.md](docs/v2/phase-2-asset-model.md)), and Phase 3 +is the deterministic **document-ingestion** plane +([docs/v2/phase-3-document-ingestion.md](docs/v2/phase-3-document-ingestion.md)). +The following later-phase items are **not** implemented; the foundation creates +extension points for them rather than building them: + +- Requirement, Business Rule, Design Decision and Acceptance Criterion + extraction; semantic document classification; document-authority inference; - Claim and Relation models; - the engineering Knowledge Graph and requirement-to-code traceability; -- PDF, DOCX and XLSX parsing; OCR; COBOL and JCL support; requirement and - business-rule extraction; +- **OCR** — image-only PDFs are *detected* and marked `needs-ocr`, never read; +- COBOL, JCL, PPTX and email-archive parsing; Jira and Confluence connectors; - cloud model providers (OpenAI, Anthropic, Bedrock, Azure, Vertex) — the runtime remains local-first with no cloud credentials; -- requirement-to-code traceability, conflict detection and branch overlays; +- conflict detection and branch overlays; +- historical (non-current-revision) document search; - webhook integration and a Bundle 2.0 artifact schema (export stays at schema 1.x); - a typed worker pool or job DAG replacing the current single-worker engine. diff --git a/docs/cli.md b/docs/cli.md index e028a6c..c4941bc 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -226,6 +226,111 @@ segment/evidence id is a typed not-found (exit `1`), never a leak. `assets_removed`, `revisions`, `segments`, `evidence`) — additive; every prior `status` key is unchanged. +### `document` + +Append and query **enterprise documents** (OpenMind v2 Phase 3). A document +becomes an Asset like any other: immutable Revisions, structural Segments, and +Evidence you can cite. See +[docs/v2/phase-3-document-ingestion.md](v2/phase-3-document-ingestion.md). + +Supported formats: Markdown, plain text / RST / AsciiDoc, HTML, CSV/TSV, DOCX, +text-based PDF, XLSX, OpenAPI (JSON/YAML), JSON Schema and SQL DDL. + +```bash +# append a document from ANYWHERE on this machine +python -m openmind.cli document add \ + --workspace p_... --path ./Requirements_v3.docx --wait --json + +# list document assets (bounded; filter by parse status or parser) +python -m openmind.cli document list --workspace p_... --json + +# one document: asset, current revision, parse summary and warnings +python -m openmind.cli document show --workspace p_... --asset a_... --json + +# a bounded STRUCTURAL outline of a revision (structure, not content) +python -m openmind.cli document outline \ + --workspace p_... --revision r_... --limit 500 --json + +# document search (exact identifiers outrank merely similar text) +python -m openmind.cli document search \ + --workspace p_... --query "REQ-NC-017" --json + +# deterministic candidate associations for one document +python -m openmind.cli document related --workspace p_... --asset a_... --json + +# code AND documents, reported separately +python -m openmind.cli knowledge search \ + --workspace p_... --query "NameCheck timeout" --json +``` + +**How to append a document.** `document add` reads a local file, snapshots its +exact bytes into the immutable content store, and enqueues a `document_ingest` +job. **The absolute path never enters the portable database** — the job payload +carries a content hash and a filename. The file does not have to live under a +registered source root; a document that does not is an *attachment*, and its +snapshot is its canonical source. + +**Duplicate and revision decisions.** `document add` reports which of five +things happened, and refuses to guess: + +| `status` | What it means | What was written | +| --- | --- | --- | +| `new_asset` | nothing matched | a new Asset + Revision | +| `revision` | the target Asset exists and the bytes differ | the next Revision | +| `duplicate` | these exact bytes are already a document Revision here | **nothing** | +| `possible_revision` | the filename collides with a DIFFERENT document | **nothing** | +| `unsupported` | no parser claims these bytes | an `unsupported` Asset | + +`possible_revision` exits **1** and hands back the commands that resolve it — +two teams' `requirements.docx` are not the same document, and merging them +because their names match is not recoverable: + +``` +--asset a_... # these bytes are a new revision of that document +--new-asset # this is a different document +--logical-key documents/... # name it yourself +``` + +`--asset`, `--logical-key` and `--new-asset` each name a *different* target for +the same bytes and are mutually exclusive (exit `2`). `--new-asset` produces a +readable deterministic key, e.g. `documents/Requirements_v3--9f2c1a3b.docx`. +`--dry-run` reports the plan and stores nothing. `--version-label` sets an +explicit label; otherwise a label is taken only from a *documented* metadata +field (OpenAPI `info.version`, the DOCX core `revision`) — never guessed from +prose or a filename. + +**How document Evidence is preserved.** Every stored block's exact text is +snapshotted as its own content-addressed blob, so `asset evidence` recovers a +historical citation **without re-running a parser** — a newer parser version +can never rewrite history. For a workspace document the current-source status is +`matches` / `changed` / `missing`; for an attachment it is `not-applicable`, +because its origin path is deliberately not tracked and `missing` would be a +false alarm. + +**Why `related` results are only candidates.** `document related` returns +**observed mentions**, each labelled `status: "candidate"` with a confidence: +`high` for an exact explicit identifier (a file path, a code symbol, a shared +requirement id), `medium` for an exact match after normalization (an API path, a +config key, a database object), `low` for embedding similarity alone. Nothing is +persisted, and no result claims the document *implements*, *refines*, *verifies* +or *contradicts* anything — that needs verification OpenMind does not yet +perform. Likewise `knowledge search` returns code and documents in separate +sections and never asserts a relationship between them. + +**Why OCR is not implemented.** An image-only PDF is *detected* and reported as +`needs-ocr`, and produces no blocks. No text is invented for it, and no result +ever claims OCR ran. Running OCR is Phase 4+. + +**Why Requirement extraction is Phase 4.** This phase ingests and normalizes the +raw document layer. Extracting Requirements, Business Rules, Design Decisions or +Acceptance Criteria — and asserting how they relate to code — requires semantic +verification, and asserting it without that would put unverifiable claims into +immutable history. + +**Why `.openmind` stays at schema 1.1.0.** The artifact bundle is a frozen +integration contract that external consumers depend on. The document model is +not exported through it, so nothing downstream changes. + ### `export` ```bash @@ -319,6 +424,27 @@ esac | `OPENMIND_SOURCELINK_EGRESS` | `0` disables on-demand GitHub source fetching | | `OPENMIND_INGEST_FREE_GPU` | `0` keeps a resident model loaded during bulk embedding | +Document parsing runs inside an explicit resource envelope. Every limit below is +overridable, and hitting one produces a `partial` parse with a warning naming +that exact limit — never a silent truncation. + +| Variable | Default | Bounds | +| --- | --- | --- | +| `OPENMIND_DOCUMENT_MAX_BYTES` | 25,000,000 | whole-document size (refused above it) | +| `OPENMIND_DOCUMENT_MAX_BLOCKS` | 20,000 | blocks per document | +| `OPENMIND_DOCUMENT_MAX_BLOCK_CHARS` | 20,000 | characters per block | +| `OPENMIND_PDF_MAX_PAGES` | 2,000 | pages read from a PDF | +| `OPENMIND_DOCX_MAX_PARAGRAPHS` | 50,000 | DOCX paragraphs | +| `OPENMIND_DOCX_MAX_TABLES` | 2,000 | DOCX tables | +| `OPENMIND_XLSX_MAX_SHEETS` | 100 | worksheets read | +| `OPENMIND_XLSX_MAX_ROWS_PER_SHEET` | 20,000 | rows per sheet | +| `OPENMIND_XLSX_MAX_CELLS` | 500,000 | cells per workbook | +| `OPENMIND_CSV_MAX_ROWS` | 50,000 | CSV/TSV rows | +| `OPENMIND_ZIP_MAX_MEMBERS` | 2,000 | members in a DOCX/XLSX package | +| `OPENMIND_ZIP_MAX_TOTAL_BYTES` | 400,000,000 | total uncompressed size | +| `OPENMIND_ZIP_MAX_MEMBER_BYTES` | 100,000,000 | one member's uncompressed size | +| `OPENMIND_ZIP_MAX_RATIO` | 200 | compression ratio (bomb detection) | + The acceptance runner sets the offline/no-egress set for every test. --- diff --git a/docs/database-migrations.md b/docs/database-migrations.md index d985ba1..e9e87c7 100644 --- a/docs/database-migrations.md +++ b/docs/database-migrations.md @@ -76,6 +76,7 @@ runner switches the connection to explicit-transaction mode | 1 | `baseline` | the schema as it stood before migrations existed: `projects`, `jobs`, `model_config`, `file_index`, `kv`, `ask_history` + its index | | 2 | `paths_sidecar` | moves legacy in-DB project paths into the machine-local sidecar and blanks `projects.paths_json` | | 3 | `asset_model` | the canonical Asset model: `assets`, `asset_revisions`, `segments`, `evidence` + their indexes. Additive — creates only new tables (all `IF NOT EXISTS`), touches no existing row | +| 4 | `document_ingestion` | document ingestion: `segments.content_blob_hash`, `jobs.payload_json`, `document_parses`, `document_index` + their indexes. Additive — two columns with defaults and two new tables | `v0003` adds the OpenMind v2 canonical content-identity model. `assets` references `projects(id)` and the whole subtree cascades on `ON DELETE CASCADE`, @@ -86,15 +87,43 @@ representable. Content bytes live in an immutable content-addressed blob store (`data//objects/…`), never in the database — the DB stores only the SHA-256 hash. See [docs/v2/phase-2-asset-model.md](v2/phase-2-asset-model.md). +`v0004` adds the document-ingestion plane. Two of its four changes deserve a +word about *why*: + +* **`segments.content_blob_hash`** — code Evidence is recovered by slicing + `[startLine, endLine]` out of the revision blob, which is safe forever because + the byte-to-line mapping never changes. A *document* block cannot be recovered + that way: re-deriving it needs a parser rerun, and a future parser version may + legitimately produce different boundaries, which would silently rewrite + history. Document segments therefore snapshot their exact text as their own + content-addressed blob. Existing code segments keep `''` and are deliberately + **not** backfilled. +* **`jobs.payload_json`** — a document import must reach the worker without an + absolute machine path in the portable database. The free `path` column already + means something different per job type, so the safe import payload (staged + blob hash, filename, requested target, import mode) gets its own column. + +`document_parses` is a derived parse projection over an immutable Revision, and +its *presence* for an Asset's current Revision is the definition of "this Asset +is a document" — a recorded fact, never an inference from the file extension. +`document_index` names the vector chunks currently live for an Asset, so a new +current Revision replaces exactly its predecessor's chunks. See +[docs/v2/phase-3-document-ingestion.md](v2/phase-3-document-ingestion.md). + +Unusually, `v0004` declares `upgrade(conn)` rather than `STATEMENTS`: SQLite has +no `ALTER TABLE ... ADD COLUMN IF NOT EXISTS`, and a plain `ADD COLUMN` raises +"duplicate column name" on a second run. Reading `PRAGMA table_info` first keeps +the whole migration idempotent. + ### Upgrading an existing database Nothing to do — open OpenMind and it migrates itself. Concretely: ```text -empty database -> v0001..v0003 create every table -> ledger records 1, 2, 3 -legacy database -> v0001 statements are all no-ops, -> ledger records 1, 2, 3 - v0002/v0003 apply additively (existing data untouched) -current database -> nothing to apply -> no writes +empty database -> v0001..v0004 create every table -> ledger records 1..4 +legacy database -> v0001 statements are all no-ops, -> ledger records 1..4 + v0002..v0004 apply additively (existing data untouched) +current database -> nothing to apply -> no writes ``` A Phase 1 database (already at v0002) upgrades to v0003 with no data loss: @@ -102,6 +131,12 @@ A Phase 1 database (already at v0002) upgrades to v0003 with no data loss: are then backfilled on the next ingestion — without re-embedding unchanged files, reusing their existing Chroma chunks. +A Phase 2 database (at v0003) upgrades to v0004 with no data loss either: +`v0004` adds two columns whose defaults are correct for every existing row, and +two tables that start empty. No project, path, job, Asset, Revision, Segment, +Evidence row, content blob, file-index row, vector collection, map, case, Ask +exchange or template selection is touched. + A legacy database is **baselined, not recreated**. `v0001` is written entirely with `CREATE TABLE IF NOT EXISTS`, so against a database that already has those tables every statement is a no-op and the runner simply records version 1. diff --git a/docs/v2/phase-2-asset-model.md b/docs/v2/phase-2-asset-model.md index 3b18b7b..284768a 100644 --- a/docs/v2/phase-2-asset-model.md +++ b/docs/v2/phase-2-asset-model.md @@ -364,6 +364,15 @@ green; delete stays responsive and the startup janitor race fix is preserved. ## 13. Deferred Phase 3+ work +> **Status update.** Phase 3 has since delivered the document-ingestion half of +> this list — DOCX/PDF/XLSX/Markdown/HTML/CSV/OpenAPI/JSON-Schema/SQL parsing, +> document Assets with block-level Evidence, document search and deterministic +> candidate association. See +> [phase-3-document-ingestion.md](phase-3-document-ingestion.md). Everything +> else below is still deferred, including OCR and requirement traceability. +> This section is left as written to record what Phase 2 itself did and did not +> do. + Intentionally **not** implemented here: PDF/DOCX/XLSX parsing, OCR pipelines, COBOL/JCL parsers, requirement/business-rule extraction, cloud LLM providers, induced Project Lens, Claim/Relation tables, Knowledge-Graph edges, From 747d594685e554d391b70064e38e2ccda0227ea4 Mon Sep 17 00:00:00 2001 From: HelloThisWorld Date: Tue, 21 Jul 2026 01:02:45 +0800 Subject: [PATCH 7/9] Phase 3 (docs): record the delivered result in the design document Adds a Result section: the full-tier acceptance numbers (36 passed, 0 failed, 1,568 individual checks), the per-suite check counts, the schema head and tool set, the five bugs the suites caught during implementation, and the two out-of-band verifications (dependency-free behaviour and the populated Phase 2 -> Phase 3 upgrade). --- docs/v2/phase-3-document-ingestion.md | 46 ++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/docs/v2/phase-3-document-ingestion.md b/docs/v2/phase-3-document-ingestion.md index 3e23165..7083a48 100644 --- a/docs/v2/phase-3-document-ingestion.md +++ b/docs/v2/phase-3-document-ingestion.md @@ -658,7 +658,51 @@ each a few kilobytes. --- -## 17. Deferred (Phase 4+) work +## 17. Result + +Implemented as designed. The full suite, every tier: + +``` +python scripts/run_acceptance.py --all --json +ok=true — 36 passed, 0 failed, 0 skipped (1,568 individual checks) +``` + +| Script | Checks | Script | Checks | +| --- | --- | --- | --- | +| verify_document_registry | 42 | verify_document_search | 79 | +| verify_document_parsers | 190 | verify_document_cli | 90 | +| verify_document_security | 74 | verify_document_adapters | 93 | +| verify_document_ingest | 102 | verify_migrations | 76 (was 64) | + +Schema head: **4**. MCP tool set: **19** (9 core + 4 Asset + 6 document), with +the first thirteen unchanged. Artifact contract: **1.1.0**, unchanged. + +Five bugs the suites caught during implementation, all fixed: + +1. `DocumentBuilder.full` was a silent early-exit signal — a parser that broke + out of its loop produced a truncated document still labelled `parsed`. +2. `decode_text` tried UTF-16 before the single-byte fallbacks, and a UTF-16 + decode of arbitrary text succeeds whenever the length is even, silently + turning `caf\xe9 latte` into CJK mojibake. +3. `cmd_document_add` mutated `ok`/`error` *after* emitting, so a + `possible_revision` printed `"ok": true` while exiting non-zero. +4. `extract_terms` fed identifier COMPONENTS back as lexical terms, so + `REQ-NC-017` also matched `REQ-NC-018` and `REQ-NC-019`. +5. A removed-then-reappeared document was reactivated but never re-indexed — + active in the database and invisible to search, which is the worst failure + mode because nothing looks wrong. + +Verified out of band as well: with `python-docx`, `pypdf`, `openpyxl` and +`defusedxml` all blocked, the artifact export still runs, the registry still +loads, Markdown/HTML/CSV still parse, and DOCX/PDF/XLSX report +`dependency_unavailable` for the *right* parser. A database populated by the +Phase 2 build and opened by this one migrates 3 → 4 with zero differences across +projects, jobs, file index, assets, revisions, segments, evidence, Ask history, +kv and project meta. + +--- + +## 18. Deferred (Phase 4+) work Explicitly **not** implemented here, and never described as implemented: From 7b49d50d1300e0be79e96aee10da3704c108398f Mon Sep 17 00:00:00 2001 From: HelloThisWorld Date: Tue, 21 Jul 2026 01:19:15 +0800 Subject: [PATCH 8/9] Fix CI: two failures caused by Phase 3, both real MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Smoke (all three OS) — `KeyError: 'file'` ------------------------------------------ The asset smoke step took assets[0] and read locator["file"]. Since Phase 3 the fixture repo's README.md and docs/GLOSSARY.md are also ingested (as DOCUMENT assets) and they sort first, so assets[0] became a document whose locator names its target in `document`, not `file`. That is correct new behaviour — document discovery is supposed to find .md files — so the fix is in the check, not the code. The step now: * selects a source-code asset explicitly, because it exists to prove the CODE asset chain and taking assets[0] would silently retarget it; * asserts the locator kind is source-range, so a future retarget fails loudly instead of passing on the wrong thing; * additionally walks EVERY asset and asserts the portable-key rule holds whichever locator kind it uses (`file` for code, `document` for documents, never absolute). Locally this now covers 10 source-range and 2 text-range locators, so both kinds are exercised. Full gate — verify_document_security, windows-traversal member name ------------------------------------------------------------------- ZipInfo.__init__ rewrites os.sep to "/", so a member written as `..\..\x` is stored with forward slashes on Windows and verbatim with backslashes on Linux. The assertion compared against the REQUESTED name, so it passed on Windows and failed on Linux. The refusal itself was always correct on both. Fixed by comparing against what the archive actually stored, plus seven direct checks on the _is_traversal predicate (including the literal-backslash form Linux keeps) so the detector is pinned independently of whatever zipfile did to the name. verify_document_security: 74 -> 81 checks. Full suite, every tier: 36 passed, 0 failed, 0 skipped. --- .github/workflows/ci.yml | 28 ++++++++++++++++++++++- tests/verify_document_security.py | 38 ++++++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77dc436..97b6f05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -330,14 +330,40 @@ jobs: [sys.executable, "-m", "openmind.cli", *args])) data = cli("asset", "list", "--workspace", ws, "--json") assert data["ok"] and data["total"] >= 1, data - aid = data["assets"][0]["id"] + # This step proves the CODE asset chain, so pick a code asset + # explicitly. Since v2 Phase 3 the fixture's README.md and + # docs/GLOSSARY.md are also ingested (as DOCUMENT assets), and they + # sort first — taking assets[0] would silently retarget this check. + code = [a for a in data["assets"] if a["asset_type"] == "source-code"] + assert code, data + aid = code[0]["id"] show = cli("asset", "show", "--workspace", ws, "--asset", aid, "--json") rid = show["asset"]["current_revision"]["id"] segs = cli("asset", "segments", "--workspace", ws, "--revision", rid, "--json") eid = segs["segments"][0]["evidence_id"] ev = cli("asset", "evidence", "--workspace", ws, "--evidence", eid, "--json") assert ev["snapshot"]["status"] == "available", ev + assert ev["locator"]["kind"] == "source-range", ev assert not ev["locator"]["file"].startswith("/"), ev + + # The portable-key rule holds for EVERY asset, whichever locator kind + # it uses: a code locator names its target in `file`, a document + # locator in `document`, and neither may ever be absolute. + for asset in data["assets"]: + rev = cli("asset", "show", "--workspace", ws, + "--asset", asset["id"], "--json")["asset"] + current = rev.get("current_revision") + if not current: + continue + seg = cli("asset", "segments", "--workspace", ws, + "--revision", current["id"], "--limit", "1", "--json") + evidence_id = seg["segments"][0]["evidence_id"] + locator = cli("asset", "evidence", "--workspace", ws, + "--evidence", evidence_id, "--json")["locator"] + key = locator.get("file") or locator.get("document") or "" + assert key, (asset["logical_key"], locator) + assert not key.startswith("/") and ":" not in key[:3], \ + (asset["logical_key"], locator) print("asset CLI smoke OK:", data["total"], "assets, evidence snapshot", ev["snapshot"]["status"]) PY diff --git a/tests/verify_document_security.py b/tests/verify_document_security.py index 8c8aea5..3442013 100644 --- a/tests/verify_document_security.py +++ b/tests/verify_document_security.py @@ -41,6 +41,18 @@ def make_zip(members): return buf.getvalue() +def stored_names(data): + """The member names as the archive ACTUALLY holds them. + + ``ZipInfo.__init__`` rewrites ``os.sep`` to ``/``, so a member written as + ``..\\..\\x`` is stored with forward slashes on Windows and verbatim with + backslashes on Linux. Asserting against the requested name would therefore + pass on one OS and fail on the other — which is exactly what happened. + """ + with zipfile.ZipFile(io.BytesIO(data)) as zf: + return zf.namelist() + + def refuses(data, code, **overrides): limits = dict(LIMITS) limits.update(overrides) @@ -59,18 +71,32 @@ def refuses(data, code, **overrides): ("/etc/shadow", "absolute posix path"), ("C:/Windows/x", "windows drive path"), ("a/../../b", "traversal in the middle")): - ok, exc = refuses(make_zip([(member, b"x")]), "zip-path-traversal") + archive = make_zip([(member, b"x")]) + ok, exc = refuses(archive, "zip-path-traversal") check(f"zip: {label} is refused", ok) - # zipfile normalizes backslashes to '/' on write, so the reported member is - # the STORED name. What matters is that the refusal names the real member. - check(f"zip: the {label} refusal names the member", - bool(exc) and str(exc.detail.get("member", "")) - == member.replace("\\", "/")) + check(f"zip: the {label} refusal names the member as stored", + bool(exc) and str(exc.detail.get("member", "")) == stored_names(archive)[0]) ok, _ = refuses(make_zip([("word/document.xml", b""), ("a/b/c.xml", b"x")]), "zip-path-traversal") check("zip: an ordinary nested member is NOT refused", not ok) +# The predicate itself, independent of what zipfile stored. On Linux a member +# name keeps its backslashes (backslash is a legal POSIX filename character), +# so the detector has to normalize them itself rather than relying on the +# archive having done it. +from openmind.documents.security import _is_traversal # noqa: E402 + +for raw, expected in ((r"..\..\windows\system32\x", True), + ("../../etc/passwd", True), + (r"C:\Windows\x", True), + ("/etc/shadow", True), + (r"a\b\c.xml", False), + ("word/document.xml", False), + ("a..b/c.xml", False)): + check(f"zip: traversal detection for {raw!r} is {expected}", + _is_traversal(raw) is expected) + # --------------------------------------------------------------------------- # 2. Expansion caps # --------------------------------------------------------------------------- From 43f49c15c0665309944f27e310a03ff9dea41c32 Mon Sep 17 00:00:00 2001 From: HelloThisWorld Date: Tue, 21 Jul 2026 01:29:46 +0800 Subject: [PATCH 9/9] Fix CI: the fixture check asserted an unachievable property MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "regenerate byte-identically" gate failed on Ubuntu for sample-requirements.docx and sample-tests.xlsx. It was not version drift — CI resolved exactly the versions used to generate them (python-docx 1.2.0, openpyxl 3.1.5, lxml 6.1.1). A DOCX/XLSX is a DEFLATE archive, and the compressed bytes depend on the platform's zlib build. Same parts in, different container bytes out. So the check was asserting something that cannot hold across operating systems, and the property it actually needed - "regenerating yields the same document" - was never being tested. The check now compares by format, because the achievable guarantee differs: PDFs written byte by byte by the generator, no compressor involved -> compared as exact bytes, unchanged OOXML unpacked and compared member by member, which is the property that matters and still catches real generator or library drift Verified both directions locally: it passes on the committed fixtures, and perturbing the generator (changing the DOCX core author) is still caught, now naming the exact differing part - "sample-requirements.docx: changed= ['docProps/core.xml']" - which the byte comparison could never have told us. Storing the archives uncompressed WOULD give true byte-identity everywhere, but it inflates sample-requirements.docx from 37 KB to 832 KB. Not a trade worth making for a test fixture. Also corrects the claim in fixtures/documents/README.md that regenerating and committing "should produce an empty diff" - true on one machine, false across platforms - and the matching claim in the generator's docstring. Nothing depends on the exact committed bytes: the tests hash the fixture at runtime and no expected hash is written down anywhere. Full suite, every tier: 36 passed, 0 failed, 0 skipped. --- .github/workflows/ci.yml | 66 ++++++++++++++++++++++++------ fixtures/documents/README.md | 35 +++++++++++----- scripts/build_document_fixtures.py | 19 ++++++--- 3 files changed, 92 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97b6f05..176a846 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,23 +137,63 @@ jobs: - name: Document adapter tests run: python scripts/run_acceptance.py --only verify_document_adapters - # The generated binary fixtures must regenerate byte-identically. If they - # do not, an "unchanged" document would mint a new Revision on every run - # and the incremental guarantee would be untestable. - - name: Document fixtures regenerate byte-identically + # The generated fixtures must regenerate to the same DOCUMENT, so an + # "unchanged" document never mints a new Revision and the incremental + # guarantee stays testable. + # + # The comparison differs by format, because the achievable guarantee does: + # + # PDFs - written byte by byte by the generator, no compressor involved, + # so they are compared as exact bytes. + # OOXML - a DOCX/XLSX is a DEFLATE archive, and the compressed bytes + # depend on the platform's zlib build. Comparing the container + # would fail on Linux for fixtures generated on Windows even + # when every part is identical. So the archive is unpacked and + # its member map is compared, which is the property that + # actually matters and catches real generator or library drift. + - name: Document fixtures regenerate to identical content run: | python - <<'PY' - import hashlib, pathlib, subprocess, sys - d = pathlib.Path("fixtures/documents") - before = {p.name: hashlib.sha256(p.read_bytes()).hexdigest() - for p in sorted(d.glob("*"))} + import pathlib, subprocess, sys, zipfile + + FIXTURES = pathlib.Path("fixtures/documents") + OOXML = {".docx", ".xlsx"} + + def content(path): + if path.suffix.lower() in OOXML: + with zipfile.ZipFile(path) as zf: + return {i.filename: zf.read(i.filename) + for i in zf.infolist()} + return path.read_bytes() + + before = {p.name: content(p) for p in sorted(FIXTURES.glob("sample-*"))} + assert before, "no fixtures found" subprocess.run([sys.executable, "scripts/build_document_fixtures.py"], check=True, capture_output=True) - after = {p.name: hashlib.sha256(p.read_bytes()).hexdigest() - for p in sorted(d.glob("*"))} - drifted = [n for n in before if before[n] != after.get(n)] - assert not drifted, f"fixtures are not reproducible: {drifted}" - print("document fixtures reproducible:", len(before), "files") + after = {p.name: content(p) for p in sorted(FIXTURES.glob("sample-*"))} + + problems = [] + for name, old in before.items(): + new = after.get(name) + if new == old: + continue + if isinstance(old, dict) and isinstance(new, dict): + # Name the exact differing part, so the failure is actionable + # rather than just "the bytes changed". + added = sorted(set(new) - set(old)) + removed = sorted(set(old) - set(new)) + changed = sorted(k for k in set(old) & set(new) + if old[k] != new[k]) + problems.append(f"{name}: added={added} removed={removed} " + f"changed={changed}") + else: + problems.append(f"{name}: bytes differ " + f"({len(old)} -> {len(new or b'')})") + assert not problems, "fixtures are not reproducible:\n " + \ + "\n ".join(problems) + print("document fixtures reproducible:", len(before), "files " + f"({sum(1 for n in before if n.endswith(tuple(OOXML)))} OOXML " + "compared by member content, the rest byte for byte)") PY - name: MCP smoke test diff --git a/fixtures/documents/README.md b/fixtures/documents/README.md index 313d055..6f6230a 100644 --- a/fixtures/documents/README.md +++ b/fixtures/documents/README.md @@ -44,15 +44,30 @@ and puts the bytes fully under our control — which is what makes "this page ha no extractable text" and "this file is encrypted" reliable test inputs instead of a library's changing idea of them. -### Why regeneration is byte-identical +### What regeneration guarantees -An unchanged document must produce **no new Revision**. A fixture whose bytes +An unchanged document must produce **no new Revision**. A fixture whose *content* drifted between runs would change its content hash and make that guarantee -untestable. So every timestamp is pinned to a fixed epoch, and the OOXML -packages are rewritten with fixed member order, fixed member timestamps and -fixed compression (`normalize_zip` in the generator) — `openpyxl` in particular -overwrites `dcterms:modified` with the wall clock during `save()`, so that field -is rewritten afterwards. - -Regenerating and committing should therefore produce an empty diff. If it does -not, the generator changed and the change is real. +untestable. So every timestamp is pinned to a fixed epoch, and the OOXML packages +are rewritten with fixed member order and fixed member timestamps +(`normalize_zip` in the generator) — `openpyxl` in particular overwrites +`dcterms:modified` with the wall clock during `save()`, so that field is +rewritten afterwards. + +The guarantee is **content**-level, and it differs by format for a reason: + +| Format | Reproducible | Why | +| --- | --- | --- | +| `.pdf` | byte for byte | the generator writes the bytes itself; no compressor is involved | +| `.docx`, `.xlsx` | member for member | a DOCX/XLSX is a DEFLATE archive, and the compressed bytes depend on the platform's zlib build. Same parts in, different container bytes out. | + +So on the **same machine**, regenerating produces an empty diff. Regenerating on +a *different platform* can legitimately change the OOXML container bytes while +every part inside is identical — that is not drift, and CI compares the member +map rather than the archive so it is not reported as such. Storing the archives +uncompressed would make them byte-identical everywhere, but it inflates +`sample-requirements.docx` from 37 KB to 832 KB, which is not a trade worth +making for a test fixture. + +Nothing depends on the exact committed bytes: the tests hash the fixture at +runtime, and no expected hash is written down anywhere. diff --git a/scripts/build_document_fixtures.py b/scripts/build_document_fixtures.py index a9ce2d1..7b43d44 100644 --- a/scripts/build_document_fixtures.py +++ b/scripts/build_document_fixtures.py @@ -16,10 +16,18 @@ WHY DETERMINISM MATTERS HERE ---------------------------- -A fixture that changes byte-for-byte between runs would make the content hash -change, and the whole point of the Asset model is that an unchanged document -produces no new Revision. So every timestamp is fixed, every id is fixed, and no -generator field is allowed to carry "now". +A fixture whose CONTENT changes between runs would change its content hash, and +the whole point of the Asset model is that an unchanged document produces no new +Revision. So every timestamp is fixed, every id is fixed, and no generator field +is allowed to carry "now". + +The achievable guarantee differs by format. The PDFs are byte-identical +everywhere, because this file writes their bytes directly. A DOCX/XLSX is a +DEFLATE archive, and the compressed container bytes depend on the platform's +zlib build — so those are reproducible member-for-member, not byte-for-byte, +across operating systems. CI compares them accordingly. Storing them +uncompressed would buy true byte-identity at the cost of inflating the DOCX from +37 KB to 832 KB, which is not worth it for a fixture. The PDFs are written by hand rather than with a PDF library: it keeps the repository free of a PDF *writing* dependency (we only need to *read* PDFs), the @@ -40,7 +48,8 @@ REPO_ROOT = Path(__file__).resolve().parent.parent FIXTURE_DIR = REPO_ROOT / "fixtures" / "documents" -#: Every generated timestamp. Fixed so a regenerated fixture is byte-identical. +#: Every generated timestamp. Fixed so a regenerated fixture has identical +#: content (see the module docstring on what "identical" means per format). EPOCH = _dt.datetime(2026, 1, 2, 3, 4, 5)