diff --git a/AGENTS.md b/AGENTS.md index bf3c7e98b..b34194c8d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -150,6 +150,10 @@ contextual-orchestrator owns model discovery and selection. from derived semantic text, while retaining the source body and meaningful list/heading nesting. A buyer-facing post view must render semantic paragraphs, not the authoring application's spacing workaround. +- Quantity HTML ``/`` and caret exponents such as `m^3` become + Unicode in derived units and React ``/`` in the post view + (ADR 0165). Never assign the body to `innerHTML`. Do not treat + `qty < 50` or a leading footnote `^1` as an exponent. - Image descriptions, OCR text, and region evidence are analysis artifacts, not buyer-facing prompt instructions. Buyer UI shows the source content and useful captions/evidence only, with provenance where appropriate. diff --git a/CHANGELOG.d/2.12.18-quantity-superscripts.md b/CHANGELOG.d/2.12.18-quantity-superscripts.md new file mode 100644 index 000000000..21505d37b --- /dev/null +++ b/CHANGELOG.d/2.12.18-quantity-superscripts.md @@ -0,0 +1,8 @@ +# 2.12.18 — Quantity superscript display + +## Fixed + +- Post popups now show cubic metres and similar quantities as superscripts + and subscripts (`12 m³`, `H₂O`) instead of flattened `m^3` or `m3`. + Semantic units store Unicode so embeddings keep the exponent. Comparison + operators and leading footnote carets stay literal (ADR 0165). diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f94a2c23..164660c48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,14 @@ All notable changes to this project are documented here. Format follows environment, so local OIDC and synthetic-data workflows resolve the same pinned dependencies as CI. +## [2.12.18] - 2026-08-24 + +### Fixed + +- Quantity superscripts and subscripts (`m³`, `H₂O`) now render as + text-level runs in the post popup and persist as Unicode in semantic + units. Comparison operators and leading footnote carets stay literal. + ## [2.12.17] - 2026-08-24 ### Added diff --git a/docs/adr/0165-quantity-script-display.md b/docs/adr/0165-quantity-script-display.md new file mode 100644 index 000000000..9b38bb6f8 --- /dev/null +++ b/docs/adr/0165-quantity-script-display.md @@ -0,0 +1,55 @@ +# ADR 0165: Render quantity superscripts as text runs, Unicode in units + +- Status: Accepted +- Date: 2026-08-22 +- Depends on: [0061](0061-post-body-character-reference-decoding.md), [0062](0062-semantic-unit-embedding.md), [0102](0102-semantic-source-unit-boundaries.md) + +## Context + +Imported posts write cubic metres and similar quantities as HTML +``/`` or as caret exponents (`m^3`). The display path converted +`` into a caret and then rendered the paragraph as a React text node, +so buyers saw `m^3` instead of a superscript. The DOM chunker dropped the +tags entirely, so embeddings received concatenated `m3`, which is a +different quantity. Comparison operators such as `qty < 50` must remain +plain text, and a leading footnote caret (`^1 …`) is not a unit exponent. + +## Decision + +- Preserve the source body. Derived semantic text maps HTML ``/`` + and quantity caret exponents onto Unicode Super/Subscript characters + (The Unicode Consortium, 2024, §22.4) so search and embeddings can tell + `m³` from `m3` without keeping markup in the unit (Cai, Yu, Wen, & Ma, + 2003). +- The buyer post view splits those Unicode (or leftover caret) runs and + renders them as React ``/`` elements. The body is never + assigned to `innerHTML` (ADR 0061). +- Only a letter, digit, or closing `)` immediately followed by `^` and a + short numeric/`n` exponent is treated as a quantity. A leading `^1` + footnote marker and comparison operators stay literal. +- Unmapped script runs keep a caret or underscore so they remain visible + rather than silently concatenating. Full formula ontology remains out of + scope; this decision covers quantity display and unit-level text. + +## Consequences + +- The post popup shows `12 m³` and `H₂O` as superscripts and subscripts. +- Newly persisted `post_content_unit` text stores Unicode quantities, so + later retrieval does not depend on HTML surviving the chunker. +- Existing concatenated `m3` units stay until re-ingestion; caret-form + units still render through the display splitter. + +## References + +Cai, D., Yu, S., Wen, J.-R., & Ma, W.-Y. (2003). *VIPS: A vision-based +page segmentation algorithm* (Microsoft Research Technical Report +MSR-TR-2003-79). Microsoft Research. + +International Organization for Standardization. (2022). *Quantities and +units — Part 1: General* (ISO 80000-1:2022). + +The Unicode Consortium. (2024). *The Unicode Standard* (Version 16.0.0). +https://www.unicode.org/versions/Unicode16.0.0/ + +WHATWG. (n.d.). *HTML living standard: The `sub` and `sup` elements*. +https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-sub-and-sup-elements diff --git a/docs/adr/README.md b/docs/adr/README.md index 9890fa083..d048dea29 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -10,7 +10,7 @@ decision from them. | Supporting document | Normative ADR | |---|---| | [`product-technical-gap-baseline.md`](../product-technical-gap-baseline.md) | Product/technical traceability projection across the ADR set; ADRs remain normative | -| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0024](0024-rankweave-fusion-fail-closed.md), [0167](0167-rankweave-ranking-channel-evidence.md) | +| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0024](0024-rankweave-fusion-fail-closed.md), [0165](0165-quantity-script-display.md), [0167](0167-rankweave-ranking-channel-evidence.md) | | [`PROV_O_IMPLEMENTATION.md`](../PROV_O_IMPLEMENTATION.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`PROV_O_IMPLEMENTATION_MATRIX.md`](../PROV_O_IMPLEMENTATION_MATRIX.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`ONTOLOGY_NAMESPACE_INVENTORY.md`](../doctoring/ONTOLOGY_NAMESPACE_INVENTORY.md) | [0157](0157-public-ontology-namespace-identity.md) | diff --git a/docs/lineage-bi-research-notes.md b/docs/lineage-bi-research-notes.md index 94f99d73d..a0f202468 100644 --- a/docs/lineage-bi-research-notes.md +++ b/docs/lineage-bi-research-notes.md @@ -207,6 +207,8 @@ Anagnostopoulos, E., Batsakis, S., & Petrakis, E. G. M. (2013). CHRONOS: A reaso Browne, W. J., Goldstein, H., & Rasbash, J. (2001). Multiple membership multiple classification (MMMC) models. *Statistical Modelling*, *1*(2), 103-124. https://doi.org/10.1177/1471082X0100100202 +Cai, D., Yu, S., Wen, J.-R., & Ma, W.-Y. (2003). *VIPS: A vision-based page segmentation algorithm* (Microsoft Research Technical Report MSR-TR-2003-79). Microsoft Research. + Chang, J., & Blei, D. M. (2009). Relational topic models for document networks. In D. van Dyk & M. Welling (Eds.), *Proceedings of the 12th International Conference on Artificial Intelligence and Statistics* (pp. 81-88). PMLR. Christen, P. (2012). *Data matching: Concepts and techniques for record linkage, entity resolution, and duplicate detection*. Springer. https://doi.org/10.1007/978-3-642-31164-2 @@ -225,6 +227,8 @@ Gildea, D., & Jurafsky, D. (2002). Automatic labeling of semantic roles. *Comput Hearst, M. A. (1997). TextTiling: Segmenting text into multi-paragraph subtopic passages. *Computational Linguistics*, *23*(1), 33-64. +International Organization for Standardization. (2022). *Quantities and units — Part 1: General* (ISO 80000-1:2022). + Lewis, P., Perez, E., Piktus, A., Petroni, F., Karpukhin, V., Goyal, N., Küttler, H., Lewis, M., Yih, W., Rocktäschel, T., Riedel, S., & Kiela, D. (2020). Retrieval-augmented generation for knowledge-intensive NLP tasks. In H. Larochelle, M. Ranzato, R. Hadsell, M. F. Balcan, & H. Lin (Eds.), *Advances in Neural Information Processing Systems* (Vol. 33, pp. 9459-9474). Curran Associates. Li, M., Lv, T., Chen, J., Cui, L., Lu, Y., Florencio, D., Zhang, C., Li, Z., & Wei, F. (2023). TrOCR: Transformer-based optical character recognition with pre-trained models. *Proceedings of the AAAI Conference on Artificial Intelligence*, *37*(11), 13094-13102. https://doi.org/10.1609/aaai.v37i11.26538 @@ -241,6 +245,8 @@ Sun, Q., Yuan, J., He, S., Guan, X., Yuan, H., Fu, X., Li, J., & Yu, P. S. (2025 Tong, H., Faloutsos, C., & Pan, J.-Y. (2006). Fast random walk with restart and its applications. *Proceedings of the Sixth International Conference on Data Mining (ICDM'06)*, 613-622. https://doi.org/10.1109/ICDM.2006.70 +The Unicode Consortium. (2024). *The Unicode Standard* (Version 16.0.0). https://www.unicode.org/versions/Unicode16.0.0/ + Wang, Q., Fu, Y., Cao, Y., Wang, S., Tian, Z., & Ding, L. (2023). *Recursively summarizing enables long-term dialogue memory in large language models*. arXiv. https://arxiv.org/abs/2308.15022 WHATWG. (2026). *HTML Living Standard — sections 4.3 (sectioning content) and 4.4 (grouping content)*. https://html.spec.whatwg.org/ @@ -379,3 +385,15 @@ summarize-and-replace older turns instead of an unbounded transcript or a hard truncation that silently drops earlier decisions. This is recorded here as the citation this feature would build on, not as a claim that conversation-level compression is implemented today. + +## Quantity scripts in source units (ADR 0165) + +Board exports write cubic metres as HTML `` or as `m^3`. Flattening +those tags concatenates `m3`, which is a different quantity, and leaving +the caret in the buyer view hides the exponent. Derived units map a short +HTML/caret exponent onto Unicode Super/Subscript characters (The Unicode +Consortium, 2024, §22.4) so embeddings keep the unit (Cai, Yu, Wen, & Ma, +2003) while the post view renders React ``/`` instead of +`innerHTML`. ISO 80000-1 treats the exponent on a unit symbol as part of +the quantity, not decoration. Comparison operators and a leading footnote +caret stay literal. Full formula ontology is still open. diff --git a/frontend/package.json b/frontend/package.json index 46a418cc9..938ac6bdd 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "2.12.17", + "version": "2.12.18", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/App.css b/frontend/src/App.css index 9c0fce272..b88ee953a 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -350,6 +350,25 @@ white-space: pre-wrap; } +.post-body-text sup, +.post-body-text sub, +.post-body-table sup, +.post-body-table sub { + font-size: 0.75em; + line-height: 0; + font-weight: 600; +} + +.post-body-text sup, +.post-body-table sup { + vertical-align: super; +} + +.post-body-text sub, +.post-body-table sub { + vertical-align: sub; +} + .post-embedded-image { margin: 0; padding: var(--post-image-padding); diff --git a/frontend/src/PostBody.test.tsx b/frontend/src/PostBody.test.tsx index 1a9b00c4a..3eaba4970 100644 --- a/frontend/src/PostBody.test.tsx +++ b/frontend/src/PostBody.test.tsx @@ -58,6 +58,101 @@ describe("PostBody", () => { expect(screen.getByText("Embedded image")).toBeInTheDocument(); }); + it("renders raw and persisted encoded non-script markup as the same inert text", () => { + const encoded = + "Keep <b>bold</b>, <sup-note>2</sup-note>, " + + "<sub:item>3</sub:item>, and <script>alert(1)</script> literal."; + const visible = + "Keep bold, 2, 3, and literal."; + const { container, rerender } = render(${encoded}

`} />); + + expect(screen.getByText(visible)).toBeInTheDocument(); + expect(container.querySelector("b")).toBeNull(); + expect(container.querySelector("script")).toBeNull(); + expect(container.querySelector("sup-note")).toBeNull(); + + rerender( + ${encoded}

`} + structureUnits={[ + { + unit_index: 0, + unit_kind_code: "plain_text", + unit_text: encoded, + indent_level: 0, + indent_source_code: "explicit", + indent_confidence: 1, + indent_evidence: "Synthetic encoded source", + }, + ]} + />, + ); + + expect(screen.getByText(visible)).toBeInTheDocument(); + expect(container.querySelector("b")).toBeNull(); + expect(container.querySelector("script")).toBeNull(); + expect(container.querySelector("sup-note")).toBeNull(); + }); + + it("renders raw and legacy persisted encoded scripts with the same semantics", () => { + const encoded = + "Volume x<sup>2</sup>, coolant H<sub>2</sub>O, and area m&#94;3."; + const { container, rerender } = render(${encoded}

`} />); + + expect([...container.querySelectorAll("sup")].map((node) => node.textContent)).toEqual([ + "2", + "3", + ]); + expect(container.querySelector("sub")?.textContent).toBe("2"); + + rerender( + ${encoded}

`} + structureUnits={[ + { + unit_index: 0, + unit_kind_code: "plain_text", + unit_text: encoded, + indent_level: 0, + indent_source_code: "explicit", + indent_confidence: 1, + indent_evidence: "Synthetic legacy persisted unit", + }, + ]} + />, + ); + + expect([...container.querySelectorAll("sup")].map((node) => node.textContent)).toEqual([ + "2", + "3", + ]); + expect(container.querySelector("sub")?.textContent).toBe("2"); + }); + + it("normalizes legacy encoded scripts in persisted table cells", () => { + const { container } = render( + , + ); + + const superscript = container.querySelector("td sup"); + expect(superscript?.textContent).toBe("3"); + expect(superscript?.closest("td")?.textContent).toBe("12 m3"); + }); + it("renders authoritative LLM structure levels for semantic list units", () => { render( { expect(screen.getByText("Before").compareDocumentPosition(screen.getByAltText("Source diagram")) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); expect(screen.getByAltText("Source diagram").compareDocumentPosition(screen.getByText("After")) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); }); + + it("renders quantity superscripts as text-level sup without using innerHTML", () => { + const { container } = render(); + + const paragraph = container.querySelector("p.post-body-text"); + const superscript = paragraph?.querySelector("sup"); + expect(superscript?.textContent).toBe("3"); + expect(paragraph?.textContent).toBe("Tank volume is 12 m3."); + }); + + it("renders caret exponents and subscripts from mixed source text", () => { + const { container } = render(); + + expect(container.querySelector("sub")?.textContent).toBe("2"); + expect(container.querySelector("sup")?.textContent).toBe("3"); + }); + + it("keeps comparison operators visible as ordinary text", () => { + render(); + + expect(screen.getByText("Need delivery if qty < 50 and price > 10.")).toBeInTheDocument(); + expect(document.querySelector("sup")).not.toBeInTheDocument(); + }); }); diff --git a/frontend/src/PostBody.tsx b/frontend/src/PostBody.tsx index 6769d0192..19100c465 100644 --- a/frontend/src/PostBody.tsx +++ b/frontend/src/PostBody.tsx @@ -1,7 +1,25 @@ -import { splitPostBody, type PostBodySegment } from "./postBodyDisplay"; +import { + decodeHtmlEntities, + splitPostBody, + splitScriptRuns, + normalizeScriptText, + type PostBodySegment, +} from "./postBodyDisplay"; import { t } from "./i18n"; import type { PostContentUnit, PostImageContent } from "./api"; -import type { ReactNode } from "react"; +import { Fragment, type ReactNode } from "react"; + +function renderStyledText(text: string): ReactNode { + return splitScriptRuns(text).map((run, index) => { + if (run.script === "super") { + return {run.text}; + } + if (run.script === "sub") { + return {run.text}; + } + return {run.text}; + }); +} function parsePipeDelimitedTable(text: string): string[][] | null { const rows = text @@ -21,14 +39,14 @@ function parsePipeDelimitedTable(text: string): string[][] | null { function renderImageText(text: string) { const rows = parsePipeDelimitedTable(text); - if (!rows) return

{text}

; + if (!rows) return

{renderStyledText(text)}

; return ( {rows.map((row, rowIndex) => ( {row.map((cell, cellIndex) => ( - + ))} ))} @@ -102,7 +120,7 @@ function renderSegment(segment: PostBodySegment, index: number, imageContent?: P : undefined } > - {segment.text} + {renderStyledText(segment.text)}

); case "image": @@ -128,8 +146,12 @@ function isStructuredTableRow(unit: PostContentUnit): boolean { * its source display is still one text segment, so ordinal matching shifts * indentation for every later unresolved unit. */ +function displayUnitText(value: string): string { + return decodeHtmlEntities(normalizeScriptText(value)); +} + function normalizedUnitText(value: string): string { - return value.replace(/\s+/g, " ").trim(); + return displayUnitText(value).replace(/\s+/g, " ").trim(); } /** @@ -220,7 +242,9 @@ function renderStructuredUnits( {rows.map((row, rowIndex) => ( {row.unit_text.split(/\s*\|\s*/).map((cell, cellIndex) => ( - + ))} ))} @@ -239,7 +263,7 @@ function renderStructuredUnits( renderSegment( { kind: "text", - text: unit.unit_text, + text: displayUnitText(unit.unit_text), ...(unit.unit_label === "footnote" || sourceText?.role === "footnote" ? { role: "footnote" as const } : {}), diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 07ca9514d..3a5a5e3af 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { splitPostBody } from "./postBodyDisplay"; +import { splitPostBody, splitScriptRuns, normalizeScriptText } from "./postBodyDisplay"; /** 1x1 transparent PNG — the same synthetic fixture the Python vision tests use. */ const TINY_PNG_B64 = @@ -47,17 +47,21 @@ describe("splitPostBody", () => { ]); }); - it("reads CSS box shorthand indentation and markerless footnotes", () => { + it("reads CSS box shorthand indentation", () => { expect( splitPostBody( '
  • Outer
' + - '
  • Nested
' + - "

*Tier 2: note

", + '
  • Nested
', ), ).toEqual([ { kind: "text", text: "Outer", indentLevel: 7 }, { kind: "text", text: "Nested", indentLevel: 10 }, - { kind: "text", text: "*Tier 2: note", role: "footnote" }, + ]); + }); + + it("does not infer a footnote from a bare marker", () => { + expect(splitPostBody("

*Synthetic list item

")).toEqual([ + { kind: "text", text: "*Synthetic list item" }, ]); }); @@ -132,4 +136,139 @@ describe("splitPostBody", () => { ); expect(JSON.stringify(segments)).not.toContain("https://example.test"); }); + + it("turns HTML and caret quantity exponents into unicode without flattening them", () => { + expect(splitPostBody("

Tank volume is 12 m3.

")).toEqual([ + { kind: "text", text: "Tank volume is 12 m³." }, + ]); + expect(splitPostBody("Tank volume is 12 m^3.")).toEqual([ + { kind: "text", text: "Tank volume is 12 m³." }, + ]); + expect(splitPostBody("Coolant is H2O at 10^{-3} M.")).toEqual([ + { kind: "text", text: "Coolant is H₂O at 10⁻³ M." }, + ]); + }); + + it("decodes HTML entities inside a sup/sub tag before mapping to unicode", () => { + // Office-tool HTML export pads content with  . The raw, + // un-decoded " 3" must not fail the all-convertible check and fall + // back to a literal caret (regression: entities weren't decoded before + // the convertibility check, matching the Python backend which unescapes + // first). + expect(splitPostBody("

Volume is 12 m 3.

")).toEqual([ + { kind: "text", text: "Volume is 12 m ³." }, + ]); + }); + + it("normalizes entity-encoded quantity syntax without leaking raw markup", () => { + expect( + splitPostBody("

Reserve 12 m^3 and x<sup>2</sup> units.

"), + ).toEqual([{ kind: "text", text: "Reserve 12 m³ and x² units." }]); + }); + + it("keeps invalid encoded script pairs literal", () => { + expect( + splitPostBody( + "

Keep x<sup>2 unmatched; x<sup/>2 self-closing; " + + "x<sup class="unit">2</sup> attributed; and " + + "x<sup>2</sub> mismatched.

", + ), + ).toEqual([ + { + kind: "text", + text: + 'Keep x2 unmatched; x2 self-closing; x2 attributed; and x2 mismatched.', + }, + ]); + }); + + it("keeps encoded non-script inline markup literal", () => { + expect(splitPostBody("

Keep <b>bold</b> literal.

")).toEqual([ + { kind: "text", text: "Keep bold literal." }, + ]); + }); + + it("keeps encoded non-script block markup literal", () => { + expect( + splitPostBody( + "

Keep <table><tr><td>grid</td></tr></table> literal.

", + ), + ).toEqual([ + { kind: "text", text: "Keep
{cell}{renderStyledText(cell)}
{cell} + {renderStyledText(displayUnitText(cell))} +
grid
literal." }, + ]); + }); + + it("normalizes nested-encoded script tags and their inner entity", () => { + expect( + splitPostBody( + "

Volume is m&lt;sup&gt;&nbsp;3&lt;/sup&gt;.

", + ), + ).toEqual([{ kind: "text", text: "Volume is m ³." }]); + }); + + it("normalizes encoded script content wrapped in encoded inline markup", () => { + expect( + splitPostBody("

x<sup><span>2</span></sup>

"), + ).toEqual([{ kind: "text", text: "x²" }]); + }); + + it("keeps encoded script-prefixed custom and namespaced tags literal", () => { + expect( + splitPostBody( + "

Keep <sup-note>2</sup-note> and <sub:item>3</sub:item> literal.

", + ), + ).toEqual([ + { + kind: "text", + text: "Keep 2 and 3 literal.", + }, + ]); + }); + + it("matches sup/sub content split across a newline", () => { + // Pretty-printed source HTML puts tag content on its own line + // (regression: the regex lacked the dotAll flag, so `.` could not cross + // the newline and the whole tag passed through unmatched, leaving a + // plain un-superscripted "3" instead of "³"). + expect(splitPostBody("

Tank volume is 12 m\n3\n.

")).toEqual([ + { kind: "text", text: "Tank volume is 12 m ³ ." }, + ]); + }); + + it("does not treat a leading footnote caret or a comparison as an exponent", () => { + expect(splitPostBody("^1 See the tank note.")).toEqual([ + { kind: "text", text: "^1 See the tank note." }, + ]); + expect(normalizeScriptText("qty < 50 and price > 10")).toBe("qty < 50 and price > 10"); + expect(splitScriptRuns("Tank volume is 12 m³.")).toEqual([ + { text: "Tank volume is 12 m" }, + { text: "3", script: "super" }, + { text: "." }, + ]); + }); + + it("keeps mixed script content as a visible fallback", () => { + expect(splitPostBody("x3a")).toEqual([{ kind: "text", text: "x^3a" }]); + }); + + it("decodes a stored superscript letter deterministically to lowercase", () => { + // "n" and "N" both encode to the same Unicode "ⁿ" (there is no distinct + // uppercase superscript N), so decoding must pick one case consistently + // rather than depending on object key iteration order (regression: used + // to always decode to uppercase because "N"/"I" were inserted after + // "n"/"i" in the forward table). + expect(splitScriptRuns("mⁿ")).toEqual([ + { text: "m" }, + { text: "n", script: "super" }, + ]); + expect(splitScriptRuns("xⁱ")).toEqual([ + { text: "x" }, + { text: "i", script: "super" }, + ]); + expect(splitPostBody("mN")).toEqual([{ kind: "text", text: "mⁿ" }]); + expect(splitScriptRuns(normalizeScriptText("mN"))).toEqual([ + { text: "m" }, + { text: "n", script: "super" }, + ]); + }); }); diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 919e8c0ca..6a4f17540 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -20,7 +20,6 @@ const BLOCK_TAG = /<\/?(?:article|blockquote|div|h[1-6]|li|ol|p|section|table|tbody|td|tfoot|th|thead|tr|ul|w:p|w:tbl|w:tr|w:tc)\b[^>]*>/gi; const WORD_INDENT_TAG = /]*\/?\s*>/gi; const LIST_ITEM_START = /^\s*(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)/; -const FOOTNOTE_START = /^\s*[*†‡](?=\S)/; const INDENT_MARKER = "\u0001lw-indent:"; const INDENT_MARKER_END = "\u0002"; const INDENT_MARKER_PATTERN = /lw-indent:(\d+)/g; @@ -40,8 +39,10 @@ export function decodeHtmlEntities(text: string): string { const decoder = document.createElement("textarea"); let decoded = text; for (let pass = 0; pass < 3; pass += 1) { - decoder.innerHTML = decoded; - const next = decoder.value; + const next = decoded.replace(/&(?:#[0-9]+|#x[0-9a-f]+|[a-z][a-z0-9]+);/gi, (entity) => { + decoder.innerHTML = entity; + return decoder.value; + }); if (next === decoded) break; decoded = next; } @@ -89,9 +90,180 @@ function indentMarker(width: number): string { return width > 0 ? `${INDENT_MARKER}${width}${INDENT_MARKER_END}` : ""; } +const SUPER_ASCII_TO_UNI: Record = { + "0": "⁰", + "1": "¹", + "2": "²", + "3": "³", + "4": "⁴", + "5": "⁵", + "6": "⁶", + "7": "⁷", + "8": "⁸", + "9": "⁹", + "+": "⁺", + "-": "⁻", + "=": "⁼", + "(": "⁽", + ")": "⁾", + n: "ⁿ", + N: "ⁿ", + i: "ⁱ", + I: "ⁱ", +}; +const SUB_ASCII_TO_UNI: Record = { + "0": "₀", + "1": "₁", + "2": "₂", + "3": "₃", + "4": "₄", + "5": "₅", + "6": "₆", + "7": "₇", + "8": "₈", + "9": "₉", + "+": "₊", + "-": "₋", + "=": "₌", + "(": "₍", + ")": "₎", + a: "ₐ", + e: "ₑ", + h: "ₕ", + i: "ᵢ", + k: "ₖ", + l: "ₗ", + m: "ₘ", + n: "ₙ", + o: "ₒ", + p: "ₚ", + s: "ₛ", + t: "ₜ", + x: "ₓ", +}; +// Two ASCII keys can map to the same Unicode character (e.g. "n" and "N" +// both produce "ⁿ"). Building the reverse table naively lets the +// last-inserted ASCII key win, so decoding always yields one fixed case +// regardless of what was actually stored. Keep the first (lowercase, since +// it is listed first above) mapping instead, so round-tripping preserves case. +function buildUnicodeToAsciiTable(table: Record): Record { + const reverse: Record = {}; + for (const [ascii, uni] of Object.entries(table)) { + if (!(uni in reverse)) reverse[uni] = ascii; + } + return reverse; +} +const SUPER_UNI_TO_ASCII = buildUnicodeToAsciiTable(SUPER_ASCII_TO_UNI); +const SUB_UNI_TO_ASCII = buildUnicodeToAsciiTable(SUB_ASCII_TO_UNI); +const CARET_EXPONENT = + /(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+-]?\d{1,3}|[nNiI])\}|([+-]?\d{1,3}|[nNiI]))/g; +const ENCODED_CARET = /&(?:amp;)*(?:#0*94|#x0*5e);/gi; +const ENCODED_LT = String.raw`&(?:amp;)*(?:lt|#0*60|#x0*3c);`; +const ENCODED_GT = String.raw`&(?:amp;)*(?:gt|#0*62|#x0*3e);`; +const ENCODED_SCRIPT_TOKEN = + `${ENCODED_LT}\\s*/?\\s*(?:sup|sub)(?=\\s|/|${ENCODED_GT})`; +const ENCODED_SCRIPT_PAIR = new RegExp( + `${ENCODED_LT}(sup|sub)${ENCODED_GT}` + + `((?:(?!${ENCODED_SCRIPT_TOKEN}).)*?)${ENCODED_LT}/\\1${ENCODED_GT}`, + "gis", +); + +function applyUnicodeScript(text: string, kind: "super" | "sub"): string { + const table = kind === "super" ? SUPER_ASCII_TO_UNI : SUB_ASCII_TO_UNI; + const values = new Set(Object.values(table)); + const compact = text.trim(); + if (!compact) return text; + if ([...compact].every((ch) => ch in table || values.has(ch) || /\s/.test(ch))) { + return [...text].map((ch) => table[ch] ?? ch).join(""); + } + const prefix = kind === "super" ? "^" : "_"; + const leading = text.match(/^\s*/)?.[0] ?? ""; + const trailing = compact.length ? text.slice(leading.length + compact.length) : ""; + return `${leading}${prefix}${compact}${trailing}`; +} + +function replaceHtmlScripts(text: string): string { + return text + .replace(/]*>(.*?)<\/sup>/gis, (_match, inner: string) => + applyUnicodeScript(decodeHtmlEntities(String(inner)).replace(/<[^>]+>/g, ""), "super"), + ) + .replace(/]*>(.*?)<\/sub>/gis, (_match, inner: string) => + applyUnicodeScript(decodeHtmlEntities(String(inner)).replace(/<[^>]+>/g, ""), "sub"), + ); +} + +function decodeScriptEntities(text: string): string { + return text + .replace( + ENCODED_SCRIPT_PAIR, + (_pair, kind: string, inner: string) => + `<${kind.toLowerCase()}>${inner}`, + ) + .replace(ENCODED_CARET, (caret) => decodeHtmlEntities(caret)); +} + +export function normalizeScriptText(text: string): string { + const withCarets = decodeScriptEntities(text).replace( + CARET_EXPONENT, + (_match, braced: string, bare: string) => applyUnicodeScript(braced || bare, "super"), + ); + return replaceHtmlScripts(withCarets); +} + +export type ScriptRun = { text: string; script?: "super" | "sub" }; + +export function splitScriptRuns(text: string): ScriptRun[] { + const runs: ScriptRun[] = []; + const push = (chunk: string, script?: "super" | "sub") => { + if (!chunk) return; + const last = runs[runs.length - 1]; + if (last && last.script === script) { + last.text += chunk; + return; + } + runs.push(script ? { text: chunk, script } : { text: chunk }); + }; + let index = 0; + while (index < text.length) { + const ch = text[index]; + if (ch in SUPER_UNI_TO_ASCII) { + let ascii = SUPER_UNI_TO_ASCII[ch]; + index += 1; + while (index < text.length && text[index] in SUPER_UNI_TO_ASCII) { + ascii += SUPER_UNI_TO_ASCII[text[index]]; + index += 1; + } + push(ascii, "super"); + continue; + } + if (ch in SUB_UNI_TO_ASCII) { + let ascii = SUB_UNI_TO_ASCII[ch]; + index += 1; + while (index < text.length && text[index] in SUB_UNI_TO_ASCII) { + ascii += SUB_UNI_TO_ASCII[text[index]]; + index += 1; + } + push(ascii, "sub"); + continue; + } + if (ch === "^" && index > 0 && /[A-Za-z0-9µμ°ΩÅåÅ)]/.test(text[index - 1])) { + const rest = text.slice(index); + const match = rest.match(/^\^(?:\{([+-]?\d{1,3}|[nNiI])\}|([+-]?\d{1,3}|[nNiI]))/); + if (match) { + push(match[1] || match[2] || "", "super"); + index += match[0].length; + continue; + } + } + push(ch); + index += 1; + } + return runs; +} + function stripHtmlTags(text: string): string { - text = text.replace(/]*>(.*?)<\/sup>/gi, "^$1"); - const withBoundaries = text + const withScripts = normalizeScriptText(text); + const withBoundaries = withScripts .replace(BREAK_TAG, "\n") .replace(BLOCK_TAG, (tag) => { if (/^<\//.test(tag)) return "\n\n"; @@ -101,8 +273,7 @@ function stripHtmlTags(text: string): string { const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => /^<\/?w:/i.test(tag) ? "" : " ", ); - const decoded = decodeHtmlEntities(withoutTags); - return decoded + return decodeHtmlEntities(withoutTags) .split("\n") .map((line) => { if (!line.trim()) return ""; @@ -209,7 +380,6 @@ function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): kind: "text", text: normalized, ...(indentLevel > 0 ? { indentLevel } : {}), - ...(FOOTNOTE_START.test(normalized) ? { role: "footnote" as const } : {}), }); } } diff --git a/lineageweave/__init__.py b/lineageweave/__init__.py index 72e8bbcab..3122f99dd 100644 --- a/lineageweave/__init__.py +++ b/lineageweave/__init__.py @@ -55,4 +55,4 @@ "sentence_excerpts", ] -__version__ = "2.12.17" +__version__ = "2.12.18" diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 6d1b7ccbc..fc66e2ec3 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -97,11 +97,11 @@ # readable and attributable as one unit. _TABLE_ROW_TAGS = frozenset({"tr", "w:tr"}) _TABLE_CELL_TAGS = frozenset({"td", "th", "w:tc"}) +_TABLE_TAGS = frozenset({"table", "w:tbl"}) _LIST_ITEM_START = re.compile( r"^(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)" ) -_FOOTNOTE_START = re.compile(r"^[*†‡](?=\S)") def _is_footnote_block(tag: str, attrs: list[tuple[str, str | None]]) -> bool: @@ -130,6 +130,139 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool: ) +# Unicode Super/Subscript blocks (The Unicode Consortium, 2024, §22.4) plus the +# Latin-1 superscript digits. Quantity display uses these so embeddings keep +# "m³" distinct from "m3" without retaining HTML in the semantic text (ADR 0165). +_SUPERSCRIPT = { + "0": "\u2070", + "1": "\u00b9", + "2": "\u00b2", + "3": "\u00b3", + "4": "\u2074", + "5": "\u2075", + "6": "\u2076", + "7": "\u2077", + "8": "\u2078", + "9": "\u2079", + "+": "\u207a", + "-": "\u207b", + "=": "\u207c", + "(": "\u207d", + ")": "\u207e", + "n": "\u207f", + "N": "\u207f", + "i": "\u2071", + "I": "\u2071", +} +_SUBSCRIPT = { + "0": "\u2080", + "1": "\u2081", + "2": "\u2082", + "3": "\u2083", + "4": "\u2084", + "5": "\u2085", + "6": "\u2086", + "7": "\u2087", + "8": "\u2088", + "9": "\u2089", + "+": "\u208a", + "-": "\u208b", + "=": "\u208c", + "(": "\u208d", + ")": "\u208e", + "a": "\u2090", + "e": "\u2091", + "h": "\u2095", + "i": "\u1d62", + "k": "\u2096", + "l": "\u2097", + "m": "\u2098", + "n": "\u2099", + "o": "\u2092", + "p": "\u209a", + "s": "\u209b", + "t": "\u209c", + "x": "\u2093", +} +_SUPERSCRIPT_VALUES = frozenset(_SUPERSCRIPT.values()) +_SUBSCRIPT_VALUES = frozenset(_SUBSCRIPT.values()) +_INLINE_SCRIPT_TAGS = frozenset({"sup", "sub"}) +_HTML_SUP = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) +_HTML_SUB = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) +_INNER_TAG = re.compile(r"<[^>]+>") +# Quantity caret after a unit/digit, not a leading footnote marker such as `^1`. +_CARET_EXPONENT = re.compile( + r"(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+\-]?\d{1,3}|[nNiI])\}|([+\-]?\d{1,3}|[nNiI]))" +) +_ENCODED_CARET = re.compile(r"&(?:amp;)*(?:#0*94|#x0*5e);", re.IGNORECASE) +_ENCODED_LT = r"&(?:amp;)*(?:lt|#0*60|#x0*3c);" +_ENCODED_GT = r"&(?:amp;)*(?:gt|#0*62|#x0*3e);" +_ENCODED_SCRIPT_TOKEN = ( + rf"{_ENCODED_LT}\s*/?\s*(?:sup|sub)(?=\s|/|{_ENCODED_GT})" +) +_ENCODED_SCRIPT_PAIR = re.compile( + rf"{_ENCODED_LT}(?Psup|sub){_ENCODED_GT}" + rf"(?P(?:(?!{_ENCODED_SCRIPT_TOKEN}).)*?)" + rf"{_ENCODED_LT}/(?P=kind){_ENCODED_GT}", + re.IGNORECASE | re.DOTALL, +) + + +def apply_unicode_script(text: str, kind: str) -> str: + """Map a short exponent/index run to Unicode, or keep a caret/underscore.""" + table = _SUPERSCRIPT if kind == "sup" else _SUBSCRIPT + values = _SUPERSCRIPT_VALUES if kind == "sup" else _SUBSCRIPT_VALUES + compact = text.strip() + if not compact: + return text + if all(ch in table or ch in values or ch.isspace() for ch in compact): + return "".join(table.get(ch, ch) for ch in text) + prefix = "^" if kind == "sup" else "_" + leading_len = len(text) - len(text.lstrip()) + trailing_len = len(text) - len(text.rstrip()) + leading = text[:leading_len] + trailing = text[len(text) - trailing_len :] if trailing_len else "" + return f"{leading}{prefix}{compact}{trailing}" + + +def _decode_html_entities(text: str) -> str: + for _ in range(3): + decoded = unescape(text) + if decoded == text: + break + text = decoded + return text + + +def _decode_script_entities(text: str) -> str: + decoded_pairs = _ENCODED_SCRIPT_PAIR.sub( + lambda match: ( + f"<{match.group('kind').lower()}>{match.group('inner')}" + f"" + ), + text, + ) + return _ENCODED_CARET.sub( + lambda match: _decode_html_entities(match.group(0)), decoded_pairs + ) + + +def _replace_html_script(match: re.Match[str], kind: str) -> str: + inner = _decode_html_entities(match.group(1)) + return apply_unicode_script(_INNER_TAG.sub("", inner), kind) + + +def normalize_script_text(text: str) -> str: + """Turn HTML/caret quantity scripts into Unicode without treating comparisons as tags.""" + replaced = _CARET_EXPONENT.sub( + lambda match: apply_unicode_script(match.group(1) or match.group(2), "sup"), + _decode_script_entities(text), + ) + replaced = _HTML_SUP.sub(lambda match: _replace_html_script(match, "sup"), replaced) + replaced = _HTML_SUB.sub(lambda match: _replace_html_script(match, "sub"), replaced) + return replaced + + def normalize_semantic_text(text: str) -> str: """Remove visual hanging-indent breaks without changing source content.""" lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") @@ -144,7 +277,7 @@ def normalize_semantic_text(text: str) -> str: normalized[-1] = f"{normalized[-1]} {stripped}" else: normalized.append(stripped) - return "\n".join(normalized).strip() + return normalize_script_text("\n".join(normalized).strip()) def _source_indent_width(text: str) -> int: @@ -343,6 +476,10 @@ def __init__(self) -> None: super().__init__() self._stack: list[tuple[str, list[str], str | None, int, bool]] = [] self._unscoped_buffer: list[str] = [] + self._script_stack: list[str] = [] + self._table_cell_counts: list[int] = [] + self._table_depth = 0 + self._table_row_depths: list[int] = [] # Each entry is ("text", str, tag_name, style) or # ("image", (mime_type, bytes), "", None) -- a single sequence in # true document order, so an image's index among its siblings @@ -351,6 +488,11 @@ def __init__(self) -> None: def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: """Collect relevant text state when an HTML start tag is encountered.""" + if tag in _TABLE_TAGS: + self._table_depth += 1 + if tag in _INLINE_SCRIPT_TAGS: + self._script_stack.append(tag) + return if tag == "img": src = next((value for name, value in attrs if name == "src" and value), None) if src: @@ -376,16 +518,30 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None self._stack[-1] = (tag_name, buffer, style, indent_width, True) return if tag in _TABLE_CELL_TAGS: - if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS and self._stack[-1][1]: - self._stack[-1][1].append(" | ") + self._script_stack.clear() + if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS: + if self._table_cell_counts[-1]: + self._stack[-1][1].append(" | ") + self._table_cell_counts[-1] += 1 return + if ( + tag in _TABLE_ROW_TAGS + and self._table_row_depths + and self._table_row_depths[-1] == self._table_depth + ): + declared_width = sum(entry[3] for entry in self._stack) + tag_name, buffer, style, _, is_footnote = self._stack.pop() + self._finish_block(tag_name, buffer, style, declared_width, is_footnote) # A rich-text editor commonly wraps a table cell in a nested

or #

. Keep that content in the open row; otherwise the nested block # closes first and destroys the row/column boundary. - if any(entry[0] in _TABLE_ROW_TAGS for entry in self._stack): + if ( + tag not in _TABLE_ROW_TAGS + and any(entry[0] in _TABLE_ROW_TAGS for entry in self._stack) + ): return if tag in _DOM_BLOCK_TAGS: - if self._stack and self._stack[-1][1]: + if tag not in _TABLE_ROW_TAGS and self._stack and self._stack[-1][1]: tag_name, buffer, style, _, is_footnote = self._stack[-1] declared_width = sum(entry[3] for entry in self._stack) self._finish_block(tag_name, buffer, style, declared_width, is_footnote) @@ -397,19 +553,42 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None self._stack.append( (tag, [], style, _declared_indent_width(tag, attrs), is_footnote) ) + if tag in _TABLE_ROW_TAGS: + self._table_cell_counts.append(0) + self._table_row_depths.append(self._table_depth) def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: """Handle self-closing block tags without losing XML indentation state.""" self.handle_starttag(tag, attrs) - if tag in _DOM_BLOCK_TAGS: + if tag in _DOM_BLOCK_TAGS or tag in _INLINE_SCRIPT_TAGS or tag in _TABLE_TAGS: self.handle_endtag(tag) def handle_endtag(self, tag: str) -> None: """Close the relevant text state when an HTML end tag is encountered.""" + if ( + tag in _TABLE_TAGS + and self._table_row_depths + and self._table_row_depths[-1] == self._table_depth + ): + declared_width = sum(entry[3] for entry in self._stack) + tag_name, buffer, style, _, is_footnote = self._stack.pop() + self._finish_block(tag_name, buffer, style, declared_width, is_footnote) + if tag in _INLINE_SCRIPT_TAGS: + if tag in self._script_stack: + while self._script_stack: + closed = self._script_stack.pop() + if closed == tag: + break + return + if tag in _TABLE_CELL_TAGS: + self._script_stack.clear() + return if tag in _DOM_BLOCK_TAGS and self._stack and self._stack[-1][0] == tag: declared_width = sum(entry[3] for entry in self._stack) tag_name, buffer, style, _, is_footnote = self._stack.pop() self._finish_block(tag_name, buffer, style, declared_width, is_footnote) + if tag in _TABLE_TAGS: + self._table_depth = max(0, self._table_depth - 1) def _finish_block( self, @@ -420,12 +599,22 @@ def _finish_block( is_footnote: bool = False, ) -> None: """Emit one block buffer, including a block closed only at EOF.""" + # An unclosed / never reaches handle_endtag, so nothing else + # pops it off _script_stack. Every block boundary (a sibling block + # opening, this block's own endtag, or EOF) routes through here, so + # clearing here stops a dangling script tag from bleeding into later, + # unrelated blocks -- mirroring how a browser would not let inline + # formatting survive a block-level boundary. + self._script_stack.clear() raw_text = "".join(buffer) + if tag_name in _TABLE_ROW_TAGS: + self._table_cell_counts.pop() + self._table_row_depths.pop() for raw_unit, source_indent in _split_dom_units(raw_text): text = normalize_semantic_text(raw_unit) if text: indent_width = declared_width + source_indent - label = "footnote" if is_footnote or _FOOTNOTE_START.match(text) else tag_name + label = "footnote" if is_footnote else tag_name self._finished.append( ( "text", @@ -447,6 +636,8 @@ def handle_data(self, data: str) -> None: text = decoded had_nbsp = "\xa0" in text text = text.replace("\xa0", " ") + if self._script_stack: + text = apply_unicode_script(text, self._script_stack[-1]) if self._stack and (text.strip() or had_nbsp): self._stack[-1][1].append(text) elif text.strip() or had_nbsp: @@ -495,15 +686,31 @@ def flush() -> None: ) +def _markdown_table_cells(line: str) -> list[str]: + """Return cells while removing only optional outer pipe delimiters.""" + cells = line.strip().split("|") + if cells and not cells[0]: + cells.pop(0) + if cells and not cells[-1]: + cells.pop() + return cells + + def _is_markdown_table_row(line: str) -> bool: """Recognize a pipe row only when it has at least two cells.""" - cells = line.strip().strip("|").split("|") - return len(cells) >= 2 and all(cell.strip() for cell in cells) + cells = _markdown_table_cells(line) + return len(cells) >= 2 and any(cell.strip() for cell in cells) + + +def _is_empty_markdown_table_row(line: str, column_count: int) -> bool: + """Recognize an all-empty row only inside an established table.""" + cells = _markdown_table_cells(line) + return len(cells) == column_count and not any(cell.strip() for cell in cells) def _render_markdown_table_row(line: str) -> str: """Keep Markdown table columns as searchable row evidence.""" - return " | ".join(cell.strip() for cell in line.strip().strip("|").split("|")) + return " | ".join(cell.strip() for cell in _markdown_table_cells(line)) def _split_plain_text_units(text: str) -> list[tuple[str, int, str]]: @@ -529,15 +736,27 @@ def flush() -> None: continue if _is_markdown_table_row(line): rows: list[str] = [] - while index < len(lines) and _is_markdown_table_row(lines[index]): - rows.append(lines[index]) + column_count = len(_markdown_table_cells(line)) + while index < len(lines): + candidate = lines[index] + established = ( + len(rows) >= 2 + and bool(_MARKDOWN_TABLE_SEPARATOR.match(rows[1])) + and len(_markdown_table_cells(rows[1])) == column_count + ) + if not _is_markdown_table_row(candidate) and not ( + established + and _is_empty_markdown_table_row(candidate, column_count) + ): + break + rows.append(candidate) index += 1 data_rows = [row for row in rows if not _MARKDOWN_TABLE_SEPARATOR.match(row)] if len(data_rows) >= 2: flush() units.extend( ( - _render_markdown_table_row(row), + normalize_semantic_text(_render_markdown_table_row(row)), _source_indent_width(row), "tr", ) diff --git a/lineageweave/post_content_normalization.py b/lineageweave/post_content_normalization.py index 196b7e7b4..000d06c40 100644 --- a/lineageweave/post_content_normalization.py +++ b/lineageweave/post_content_normalization.py @@ -42,6 +42,7 @@ _HTML_OPEN_TAG = re.compile( r"<\s*/?\s*(?:article|section|nav|aside|header|footer|div|p|li|td|th|tr|" r"table|blockquote|h[1-6]|img|br|hr|ul|ol|span|strong|em|b|i|u|a|" + r"sup|sub|" r"html|body|head|style|script|font|center|pre)\b", re.IGNORECASE, ) diff --git a/pyproject.toml b/pyproject.toml index 06e27ac93..9a3975230 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lineageweave" -version = "2.12.17" +version = "2.12.18" description = "Reconstructs git-branch-style lineage DAGs from scattered short records using multi-channel score fusion and LLM adjudication." readme = "README.md" license = { text = "MIT" } diff --git a/tests/test_chunking.py b/tests/test_chunking.py index d37a300cc..b7be87013 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -7,6 +7,7 @@ chunk_by_paragraph, chunk_by_sentence, chunk_by_source_body, + normalize_script_text, normalize_semantic_text, ) @@ -104,11 +105,88 @@ def test_chunk_by_dom_keeps_nested_table_cell_blocks_in_their_row() -> None: assert [(chunk.label, chunk.text) for chunk in chunks] == [("tr", "No. | Company")] -def test_chunk_by_dom_labels_markerless_footnotes() -> None: - chunks = chunk_by_dom("

Body text

*Tier 2: follow-up note

") +def test_chunk_by_dom_preserves_empty_table_cells() -> None: + chunks = chunk_by_dom( + "
Synthetic item
" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "| Synthetic item |") + ] + + +def test_chunk_by_dom_preserves_self_closing_empty_table_cells() -> None: + html_chunks = chunk_by_dom( + "
LeftRight
" + ) + word_chunks = chunk_by_dom( + "LeftRight" + ) + + assert [(chunk.label, chunk.text) for chunk in html_chunks] == [ + ("tr", "Left | | Right") + ] + assert [(chunk.label, chunk.text) for chunk in word_chunks] == [ + ("w:tr", "Left | | Right") + ] + + +def test_chunk_by_dom_scopes_cell_positions_to_nested_table_rows() -> None: + chunks = chunk_by_dom( + "
Outer left" + "
Inner leftInner right
" + "		Outer right
" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Inner left | Inner right"), + ("tr", "Outer left | Outer right"), + ] + + +def test_chunk_by_dom_implicitly_closes_sibling_rows_at_the_same_table_depth() -> None: + chunks = chunk_by_dom( + "" + "
First leftFirst right
Second leftSecond right
" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "First left | First right"), + ("tr", "Second left | Second right"), + ] + + +def test_chunk_by_dom_closes_an_unclosed_row_at_the_table_boundary() -> None: + """A malformed final row still emits before its table closes.""" + + chunks = chunk_by_dom( + "
Only leftOnly right
" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Only left | Only right") + ] + + +def test_chunk_by_dom_closes_sibling_rows_past_an_unclosed_cell_block() -> None: + """Malformed inline cell markup cannot displace its owning row.""" + + chunks = chunk_by_dom( + "" + "
First left
First right
Second leftSecond right
" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "First left | First right"), + ("tr", "Second left | Second right"), + ] + + +def test_chunk_by_dom_does_not_infer_a_footnote_from_a_bare_marker() -> None: + chunks = chunk_by_dom("

Body text

*Synthetic list item

") assert [(chunk.label, chunk.text) for chunk in chunks] == [ ("p", "Body text"), - ("footnote", "*Tier 2: follow-up note"), + ("p", "*Synthetic list item"), ] @@ -124,7 +202,7 @@ def test_chunk_by_dom_labels_html_and_word_footnote_markup() -> None: assert [(chunk.label, chunk.text) for chunk in chunks] == [ ("p", "Body text"), ("footnote", "HTML footnote body"), - ("footnote", "1 Word footnote body"), + ("footnote", "¹ Word footnote body"), ] @@ -219,7 +297,7 @@ def test_chunk_by_source_body_splits_plain_lists_and_markdown_tables() -> None: | Field | Value | | --- | --- | -| Owner | Buyer | +| Volume | 12 m^3 | """ chunks = chunk_by_source_body(body) @@ -228,10 +306,74 @@ def test_chunk_by_source_body_splits_plain_lists_and_markdown_tables() -> None: ("", "1. Background continuation stays with the first item."), ("", "2. Decision"), ("tr", "Field | Value"), - ("tr", "Owner | Buyer"), + ("tr", "Volume | 12 m³"), ] +def test_chunk_by_source_body_normalizes_entity_encoded_quantity_scripts() -> None: + chunks = chunk_by_source_body( + "Reserve 12 m^3, x<sup>2</sup>, and H<sub>2</sub>O." + ) + + assert [chunk.text for chunk in chunks] == ["Reserve 12 m³, x², and H₂O."] + + +def test_chunk_by_source_body_keeps_invalid_encoded_script_pairs_literal() -> None: + bodies = ( + "Keep x<sup>2 unmatched.", + "Keep x<sup/>2 self-closing.", + "Keep x<sup class="unit">2</sup> attributed.", + "Keep x<sup>2</sub> mismatched.", + ) + + assert [chunk_by_source_body(body)[0].text for body in bodies] == list(bodies) + combined = " ".join(bodies) + assert chunk_by_source_body(combined)[0].text == combined + + +def test_chunk_by_source_body_keeps_encoded_non_script_markup_inert() -> None: + body = ( + "Keep <b>bold</b>, <sup-note>2</sup-note>, " + "and <script>alert(1)</script> literal." + ) + + chunks = chunk_by_source_body(body) + + assert [(chunk.unit_type, chunk.text) for chunk in chunks] == [("plain_text", body)] + + +def test_chunk_by_source_body_preserves_empty_markdown_table_cells() -> None: + chunks = chunk_by_source_body( + "| Key | Value | State |\n" + "| --- | --- | --- |\n" + "| A | | Open |" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Key | Value | State"), + ("tr", "A | | Open"), + ] + + +def test_chunk_by_source_body_keeps_contextual_all_empty_markdown_rows() -> None: + chunks = chunk_by_source_body( + "| Key | Value | State |\n" + "| --- | --- | --- |\n" + "| | | |" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Key | Value | State"), + ("tr", "| |"), + ] + + +def test_chunk_by_source_body_does_not_promote_a_standalone_empty_pipe_line() -> None: + chunks = chunk_by_source_body("| | |") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [("", "| | |")] + + def test_chunk_by_dom_joins_visual_continuation_lines_but_keeps_list_items() -> None: html = ( '

1. 배경
' @@ -410,3 +552,63 @@ def test_chunk_by_dom_preserves_style_per_block_independently() -> None: assert chunks[0].style == "color:blue" assert chunks[1].style is None + + +def test_normalize_script_text_maps_quantity_exponents_and_leaves_comparisons() -> None: + assert normalize_script_text("Tank volume is 12 m3.") == "Tank volume is 12 m³." + assert normalize_script_text("Tank volume is 12 m^3.") == "Tank volume is 12 m³." + assert normalize_script_text("Coolant is H2O.") == "Coolant is H₂O." + assert normalize_script_text("x ") == "x " + assert normalize_script_text("qty < 50 and price > 10") == "qty < 50 and price > 10" + assert normalize_script_text("^1 See the tank note.") == "^1 See the tank note." + + +def test_normalize_script_text_keeps_mixed_script_content_as_a_visible_fallback() -> None: + assert normalize_script_text("x3a") == "x^3a" + + +def test_normalize_script_text_decodes_nested_inline_markup_before_stripping() -> None: + assert normalize_script_text("x<span>2</span>") == "x²" + + +def test_chunk_by_dom_keeps_html_quantity_scripts_as_unicode() -> None: + chunks = chunk_by_dom("

Tank volume is 12 m3 of H2O.

") + + assert [chunk.text for chunk in chunks] == ["Tank volume is 12 m³ of H₂O."] + + +def test_chunk_by_dom_normalizes_entity_encoded_quantity_scripts() -> None: + chunks = chunk_by_dom( + "

Reserve 12 m^3 and x<sup>2</sup> units.

" + ) + + assert [chunk.text for chunk in chunks] == ["Reserve 12 m³ and x² units."] + + +def test_chunk_by_dom_unclosed_sup_does_not_cross_table_cells() -> None: + chunks = chunk_by_dom("
m3Acme Corp
") + + assert [chunk.text for chunk in chunks] == ["m³ | Acme Corp"] + + +def test_chunk_by_dom_unclosed_sup_does_not_corrupt_later_paragraphs() -> None: + """A malformed, never-closed must not leak its script context into + every later block. HTMLParser (unlike a browser) does not implicitly + close an unclosed inline tag at a block boundary, so a naive + _script_stack would otherwise stay "open" for the rest of the document.""" + html = ( + "

Tank volume is 12 m3

" + "

Unrelated paragraph mentions n2 and o2 plainly.

" + ) + chunks = chunk_by_dom(html) + + assert [chunk.text for chunk in chunks] == [ + "Tank volume is 12 m³", + "Unrelated paragraph mentions n2 and o2 plainly.", + ] + + +def test_chunk_by_source_body_maps_plain_caret_quantities() -> None: + chunks = chunk_by_source_body("Reserve 12 m^3 and 10^{-3} M stock.") + + assert chunks[0].text == "Reserve 12 m³ and 10⁻³ M stock." diff --git a/tests/test_post_content_normalization.py b/tests/test_post_content_normalization.py index 0beead6f4..3c7c7c0d7 100644 --- a/tests/test_post_content_normalization.py +++ b/tests/test_post_content_normalization.py @@ -375,6 +375,13 @@ def test_comparison_operators_in_plain_text_are_not_treated_as_html() -> None: assert result.formatting_hints == () +def test_quantity_superscripts_normalize_to_unicode_for_embeddings() -> None: + html = normalize_post_body("

Tank volume is 12 m3.

") + assert html.text == "Tank volume is 12 m³." + caret = normalize_post_body("Tank volume is 12 m^3.") + assert caret.text == "Tank volume is 12 m³." + + def test_image_gets_an_explicit_placeholder_when_no_vision_client_is_available() -> None: b64 = base64.b64encode(_PNG_1X1).decode("ascii") html = f'' diff --git a/uv.lock b/uv.lock index 651a179f7..112d46840 100644 --- a/uv.lock +++ b/uv.lock @@ -454,7 +454,7 @@ wheels = [ [[package]] name = "lineageweave" -version = "2.12.17" +version = "2.12.18" source = { editable = "." } dependencies = [ { name = "certifi" },