diff --git a/AGENTS.md b/AGENTS.md index cb2da918a..151568bcb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -150,7 +150,7 @@ contextual-orchestrator owns model discovery and selection. paragraphs, not the authoring application's spacing workaround. - Quantity HTML ``/`` and caret exponents such as `m^3` become Unicode in derived units and React ``/`` in the post view - (ADR 0119). Never assign the body to `innerHTML`. Do not treat + (ADR 0165). Never assign the body to `innerHTML`. Do not treat `qty < 50` or a leading footnote `^1` as an exponent. - Image descriptions, OCR text, and region evidence are analysis artifacts, not buyer-facing prompt instructions. Buyer UI shows the source content and diff --git a/CHANGELOG.d/2.12.9-quantity-superscripts.md b/CHANGELOG.d/2.12.9-quantity-superscripts.md index b2d3e3541..13498dbe4 100644 --- a/CHANGELOG.d/2.12.9-quantity-superscripts.md +++ b/CHANGELOG.d/2.12.9-quantity-superscripts.md @@ -5,4 +5,4 @@ - Post popups now show cubic metres and similar quantities as superscripts and subscripts (`12 m³`, `H₂O`) instead of flattened `m^3` or `m3`. Semantic units store Unicode so embeddings keep the exponent. Comparison - operators and leading footnote carets stay literal (ADR 0119). + operators and leading footnote carets stay literal (ADR 0165). diff --git a/docs/adr/0119-quantity-script-display.md b/docs/adr/0165-quantity-script-display.md similarity index 97% rename from docs/adr/0119-quantity-script-display.md rename to docs/adr/0165-quantity-script-display.md index fdb902daf..9b38bb6f8 100644 --- a/docs/adr/0119-quantity-script-display.md +++ b/docs/adr/0165-quantity-script-display.md @@ -1,4 +1,4 @@ -# ADR 0119: Render quantity superscripts as text runs, Unicode in units +# ADR 0165: Render quantity superscripts as text runs, Unicode in units - Status: Accepted - Date: 2026-08-22 diff --git a/docs/adr/README.md b/docs/adr/README.md index 6cb58b93c..b6819594f 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -10,7 +10,7 @@ decision from them. | Supporting document | Normative ADR | |---|---| | [`product-technical-gap-baseline.md`](../product-technical-gap-baseline.md) | Product/technical traceability projection across the ADR set; ADRs remain normative | -| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0119](0119-quantity-script-display.md) | +| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0165](0165-quantity-script-display.md) | | [`PROV_O_IMPLEMENTATION.md`](../PROV_O_IMPLEMENTATION.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`PROV_O_IMPLEMENTATION_MATRIX.md`](../PROV_O_IMPLEMENTATION_MATRIX.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`image-content-schema.md`](../image-content-schema.md) | [0066](0066-position-preserving-image-content.md) | diff --git a/docs/lineage-bi-research-notes.md b/docs/lineage-bi-research-notes.md index a3fe9e3ba..a0f202468 100644 --- a/docs/lineage-bi-research-notes.md +++ b/docs/lineage-bi-research-notes.md @@ -386,7 +386,7 @@ hard truncation that silently drops earlier decisions. This is recorded here as the citation this feature would build on, not as a claim that conversation-level compression is implemented today. -## Quantity scripts in source units (ADR 0119) +## Quantity scripts in source units (ADR 0165) Board exports write cubic metres as HTML `` or as `m^3`. Flattening those tags concatenates `m3`, which is a different quantity, and leaving diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 3904af44e..90c84ceeb 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -5,7 +5,7 @@ - **Table Parsing**: `post=00505695-3e61-1fd1-80c6-86bb61c8ddc5` completely fails at parsing tables. - **Indentation**: Incorrect indentation rendering in `post=00505695-7571-1fd1-83c3-d521b187ad5b` and `post=00505695-3e61-1fd1-83c0-497b3c1c455e`. - **Image/Table OCR**: `post=00505695-7571-1fd1-83dd-3d22a61a5734` fails text recognition for tables inside images, markdown parsing fails, and image OCR description is too shallow for Ontology & Semantics. -- **Math/Superscripts**: (Display/unit text, ADR 0119) Quantity HTML +- **Math/Superscripts**: (Display/unit text, ADR 0165) Quantity HTML ``/`` and caret exponents such as `m^3` now render as superscripts and persist as Unicode in semantic units. Full formula ontology grammar remains open. diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index c974f3dfa..20e266122 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -47,17 +47,21 @@ describe("splitPostBody", () => { ]); }); - it("reads CSS box shorthand indentation and markerless footnotes", () => { + it("reads CSS box shorthand indentation", () => { expect( splitPostBody( '
  • Outer
' + - '
  • Nested
' + - "

*Tier 2: note

", + '
  • Nested
', ), ).toEqual([ { kind: "text", text: "Outer", indentLevel: 7 }, { kind: "text", text: "Nested", indentLevel: 10 }, - { kind: "text", text: "*Tier 2: note", role: "footnote" }, + ]); + }); + + it("does not infer a footnote from a bare marker", () => { + expect(splitPostBody("

*Synthetic list item

")).toEqual([ + { kind: "text", text: "*Synthetic list item" }, ]); }); @@ -156,6 +160,55 @@ describe("splitPostBody", () => { ]); }); + it("normalizes entity-encoded quantity syntax without leaking raw markup", () => { + expect( + splitPostBody("

Reserve 12 m^3 and x<sup>2</sup> units.

"), + ).toEqual([{ kind: "text", text: "Reserve 12 m³ and x² units." }]); + }); + + it("keeps encoded non-script inline markup literal", () => { + expect(splitPostBody("

Keep <b>bold</b> literal.

")).toEqual([ + { kind: "text", text: "Keep bold literal." }, + ]); + }); + + it("keeps encoded non-script block markup literal", () => { + expect( + splitPostBody( + "

Keep <table><tr><td>grid</td></tr></table> literal.

", + ), + ).toEqual([ + { kind: "text", text: "Keep
grid
literal." }, + ]); + }); + + it("normalizes nested-encoded script tags and their inner entity", () => { + expect( + splitPostBody( + "

Volume is m&lt;sup&gt;&nbsp;3&lt;/sup&gt;.

", + ), + ).toEqual([{ kind: "text", text: "Volume is m ³." }]); + }); + + it("normalizes encoded script content wrapped in encoded inline markup", () => { + expect( + splitPostBody("

x<sup><span>2</span></sup>

"), + ).toEqual([{ kind: "text", text: "x²" }]); + }); + + it("keeps encoded script-prefixed custom and namespaced tags literal", () => { + expect( + splitPostBody( + "

Keep <sup-note>2</sup-note> and <sub:item>3</sub:item> literal.

", + ), + ).toEqual([ + { + kind: "text", + text: "Keep 2 and 3 literal.", + }, + ]); + }); + it("matches sup/sub content split across a newline", () => { // Pretty-printed source HTML puts tag content on its own line // (regression: the regex lacked the dotAll flag, so `.` could not cross @@ -178,6 +231,10 @@ describe("splitPostBody", () => { ]); }); + it("keeps mixed script content as a visible fallback", () => { + expect(splitPostBody("x3a")).toEqual([{ kind: "text", text: "x^3a" }]); + }); + it("decodes a stored superscript letter deterministically to lowercase", () => { // "n" and "N" both encode to the same Unicode "ⁿ" (there is no distinct // uppercase superscript N), so decoding must pick one case consistently diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 2773eadf2..f037a8f1a 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -20,7 +20,6 @@ const BLOCK_TAG = /<\/?(?:article|blockquote|div|h[1-6]|li|ol|p|section|table|tbody|td|tfoot|th|thead|tr|ul|w:p|w:tbl|w:tr|w:tc)\b[^>]*>/gi; const WORD_INDENT_TAG = /]*\/?\s*>/gi; const LIST_ITEM_START = /^\s*(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)/; -const FOOTNOTE_START = /^\s*[*†‡](?=\S)/; const INDENT_MARKER = "\u0001lw-indent:"; const INDENT_MARKER_END = "\u0002"; const INDENT_MARKER_PATTERN = /lw-indent:(\d+)/g; @@ -156,6 +155,9 @@ const SUPER_UNI_TO_ASCII = buildUnicodeToAsciiTable(SUPER_ASCII_TO_UNI); const SUB_UNI_TO_ASCII = buildUnicodeToAsciiTable(SUB_ASCII_TO_UNI); const CARET_EXPONENT = /(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+-]?\d{1,3}|[nNiI])\}|([+-]?\d{1,3}|[nNiI]))/g; +const ENCODED_CARET = /&(?:amp;)*(?:#0*94|#x0*5e);/gi; +const ENCODED_SCRIPT_TAG = + /&(?:amp;)*(?:lt|#0*60|#x0*3c);\s*\/?\s*(?:sup|sub)(?=\s|\/|&(?:amp;)*(?:gt|#0*62|#x0*3e);).*?&(?:amp;)*(?:gt|#0*62|#x0*3e);/gis; function applyUnicodeScript(text: string, kind: "super" | "sub"): string { const table = kind === "super" ? SUPER_ASCII_TO_UNI : SUB_ASCII_TO_UNI; @@ -174,17 +176,25 @@ function applyUnicodeScript(text: string, kind: "super" | "sub"): string { function replaceHtmlScripts(text: string): string { return text .replace(/]*>(.*?)<\/sup>/gis, (_match, inner: string) => - applyUnicodeScript(decodeHtmlEntities(String(inner).replace(/<[^>]+>/g, "")), "super"), + applyUnicodeScript(decodeHtmlEntities(String(inner)).replace(/<[^>]+>/g, ""), "super"), ) .replace(/]*>(.*?)<\/sub>/gis, (_match, inner: string) => - applyUnicodeScript(decodeHtmlEntities(String(inner).replace(/<[^>]+>/g, "")), "sub"), + applyUnicodeScript(decodeHtmlEntities(String(inner)).replace(/<[^>]+>/g, ""), "sub"), ); } +function decodeScriptEntities(text: string): string { + return text + .replace(ENCODED_SCRIPT_TAG, (tag) => decodeHtmlEntities(tag)) + .replace(ENCODED_CARET, (caret) => decodeHtmlEntities(caret)); +} + export function normalizeScriptText(text: string): string { - return replaceHtmlScripts(text).replace(CARET_EXPONENT, (_match, braced: string, bare: string) => - applyUnicodeScript(braced || bare, "super"), + const withCarets = decodeScriptEntities(text).replace( + CARET_EXPONENT, + (_match, braced: string, bare: string) => applyUnicodeScript(braced || bare, "super"), ); + return replaceHtmlScripts(withCarets); } export type ScriptRun = { text: string; script?: "super" | "sub" }; @@ -239,7 +249,7 @@ export function splitScriptRuns(text: string): ScriptRun[] { } function stripHtmlTags(text: string): string { - const withScripts = replaceHtmlScripts(text); + const withScripts = normalizeScriptText(text); const withBoundaries = withScripts .replace(BREAK_TAG, "\n") .replace(BLOCK_TAG, (tag) => { @@ -250,8 +260,7 @@ function stripHtmlTags(text: string): string { const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => /^<\/?w:/i.test(tag) ? "" : " ", ); - const decoded = decodeHtmlEntities(withoutTags); - return normalizeScriptText(decoded) + return decodeHtmlEntities(withoutTags) .split("\n") .map((line) => { if (!line.trim()) return ""; @@ -358,7 +367,6 @@ function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): kind: "text", text: normalized, ...(indentLevel > 0 ? { indentLevel } : {}), - ...(FOOTNOTE_START.test(normalized) ? { role: "footnote" as const } : {}), }); } } diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 2caf21e0e..511634ff7 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -97,11 +97,11 @@ # readable and attributable as one unit. _TABLE_ROW_TAGS = frozenset({"tr", "w:tr"}) _TABLE_CELL_TAGS = frozenset({"td", "th", "w:tc"}) +_TABLE_TAGS = frozenset({"table", "w:tbl"}) _LIST_ITEM_START = re.compile( r"^(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)" ) -_FOOTNOTE_START = re.compile(r"^[*†‡](?=\S)") def _is_footnote_block(tag: str, attrs: list[tuple[str, str | None]]) -> bool: @@ -132,7 +132,7 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool: # Unicode Super/Subscript blocks (The Unicode Consortium, 2024, §22.4) plus the # Latin-1 superscript digits. Quantity display uses these so embeddings keep -# "m³" distinct from "m3" without retaining HTML in the semantic text (ADR 0119). +# "m³" distinct from "m3" without retaining HTML in the semantic text (ADR 0165). _SUPERSCRIPT = { "0": "\u2070", "1": "\u00b9", @@ -214,23 +214,24 @@ def apply_unicode_script(text: str, kind: str) -> str: def _replace_html_script(match: re.Match[str], kind: str) -> str: - inner = _INNER_TAG.sub("", match.group(1)) + inner = match.group(1) for _ in range(3): decoded = unescape(inner) if decoded == inner: break inner = decoded - return apply_unicode_script(inner, kind) + return apply_unicode_script(_INNER_TAG.sub("", inner), kind) def normalize_script_text(text: str) -> str: """Turn HTML/caret quantity scripts into Unicode without treating comparisons as tags.""" - replaced = _HTML_SUP.sub(lambda match: _replace_html_script(match, "sup"), text) - replaced = _HTML_SUB.sub(lambda match: _replace_html_script(match, "sub"), replaced) - return _CARET_EXPONENT.sub( + replaced = _CARET_EXPONENT.sub( lambda match: apply_unicode_script(match.group(1) or match.group(2), "sup"), - replaced, + text, ) + replaced = _HTML_SUP.sub(lambda match: _replace_html_script(match, "sup"), replaced) + replaced = _HTML_SUB.sub(lambda match: _replace_html_script(match, "sub"), replaced) + return replaced def normalize_semantic_text(text: str) -> str: @@ -447,6 +448,9 @@ def __init__(self) -> None: self._stack: list[tuple[str, list[str], str | None, int, bool]] = [] self._unscoped_buffer: list[str] = [] self._script_stack: list[str] = [] + self._table_cell_counts: list[int] = [] + self._table_depth = 0 + self._table_row_depths: list[int] = [] # Each entry is ("text", str, tag_name, style) or # ("image", (mime_type, bytes), "", None) -- a single sequence in # true document order, so an image's index among its siblings @@ -455,6 +459,8 @@ def __init__(self) -> None: def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: """Collect relevant text state when an HTML start tag is encountered.""" + if tag in _TABLE_TAGS: + self._table_depth += 1 if tag in _INLINE_SCRIPT_TAGS: self._script_stack.append(tag) return @@ -484,16 +490,29 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None return if tag in _TABLE_CELL_TAGS: self._script_stack.clear() - if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS and self._stack[-1][1]: - self._stack[-1][1].append(" | ") + if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS: + if self._table_cell_counts[-1]: + self._stack[-1][1].append(" | ") + self._table_cell_counts[-1] += 1 return + if ( + tag in _TABLE_ROW_TAGS + and self._table_row_depths + and self._table_row_depths[-1] == self._table_depth + ): + declared_width = sum(entry[3] for entry in self._stack) + tag_name, buffer, style, _, is_footnote = self._stack.pop() + self._finish_block(tag_name, buffer, style, declared_width, is_footnote) # A rich-text editor commonly wraps a table cell in a nested

or #

. Keep that content in the open row; otherwise the nested block # closes first and destroys the row/column boundary. - if any(entry[0] in _TABLE_ROW_TAGS for entry in self._stack): + if ( + tag not in _TABLE_ROW_TAGS + and any(entry[0] in _TABLE_ROW_TAGS for entry in self._stack) + ): return if tag in _DOM_BLOCK_TAGS: - if self._stack and self._stack[-1][1]: + if tag not in _TABLE_ROW_TAGS and self._stack and self._stack[-1][1]: tag_name, buffer, style, _, is_footnote = self._stack[-1] declared_width = sum(entry[3] for entry in self._stack) self._finish_block(tag_name, buffer, style, declared_width, is_footnote) @@ -505,15 +524,26 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None self._stack.append( (tag, [], style, _declared_indent_width(tag, attrs), is_footnote) ) + if tag in _TABLE_ROW_TAGS: + self._table_cell_counts.append(0) + self._table_row_depths.append(self._table_depth) def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: """Handle self-closing block tags without losing XML indentation state.""" self.handle_starttag(tag, attrs) - if tag in _DOM_BLOCK_TAGS or tag in _INLINE_SCRIPT_TAGS: + if tag in _DOM_BLOCK_TAGS or tag in _INLINE_SCRIPT_TAGS or tag in _TABLE_TAGS: self.handle_endtag(tag) def handle_endtag(self, tag: str) -> None: """Close the relevant text state when an HTML end tag is encountered.""" + if ( + tag in _TABLE_TAGS + and self._table_row_depths + and self._table_row_depths[-1] == self._table_depth + ): + declared_width = sum(entry[3] for entry in self._stack) + tag_name, buffer, style, _, is_footnote = self._stack.pop() + self._finish_block(tag_name, buffer, style, declared_width, is_footnote) if tag in _INLINE_SCRIPT_TAGS: if tag in self._script_stack: while self._script_stack: @@ -521,10 +551,15 @@ def handle_endtag(self, tag: str) -> None: if closed == tag: break return + if tag in _TABLE_CELL_TAGS: + self._script_stack.clear() + return if tag in _DOM_BLOCK_TAGS and self._stack and self._stack[-1][0] == tag: declared_width = sum(entry[3] for entry in self._stack) tag_name, buffer, style, _, is_footnote = self._stack.pop() self._finish_block(tag_name, buffer, style, declared_width, is_footnote) + if tag in _TABLE_TAGS: + self._table_depth = max(0, self._table_depth - 1) def _finish_block( self, @@ -543,11 +578,14 @@ def _finish_block( # formatting survive a block-level boundary. self._script_stack.clear() raw_text = "".join(buffer) + if tag_name in _TABLE_ROW_TAGS: + self._table_cell_counts.pop() + self._table_row_depths.pop() for raw_unit, source_indent in _split_dom_units(raw_text): text = normalize_semantic_text(raw_unit) if text: indent_width = declared_width + source_indent - label = "footnote" if is_footnote or _FOOTNOTE_START.match(text) else tag_name + label = "footnote" if is_footnote else tag_name self._finished.append( ( "text", @@ -619,15 +657,31 @@ def flush() -> None: ) +def _markdown_table_cells(line: str) -> list[str]: + """Return cells while removing only optional outer pipe delimiters.""" + cells = line.strip().split("|") + if cells and not cells[0]: + cells.pop(0) + if cells and not cells[-1]: + cells.pop() + return cells + + def _is_markdown_table_row(line: str) -> bool: """Recognize a pipe row only when it has at least two cells.""" - cells = line.strip().strip("|").split("|") - return len(cells) >= 2 and all(cell.strip() for cell in cells) + cells = _markdown_table_cells(line) + return len(cells) >= 2 and any(cell.strip() for cell in cells) + + +def _is_empty_markdown_table_row(line: str, column_count: int) -> bool: + """Recognize an all-empty row only inside an established table.""" + cells = _markdown_table_cells(line) + return len(cells) == column_count and not any(cell.strip() for cell in cells) def _render_markdown_table_row(line: str) -> str: """Keep Markdown table columns as searchable row evidence.""" - return " | ".join(cell.strip() for cell in line.strip().strip("|").split("|")) + return " | ".join(cell.strip() for cell in _markdown_table_cells(line)) def _split_plain_text_units(text: str) -> list[tuple[str, int, str]]: @@ -653,8 +707,20 @@ def flush() -> None: continue if _is_markdown_table_row(line): rows: list[str] = [] - while index < len(lines) and _is_markdown_table_row(lines[index]): - rows.append(lines[index]) + column_count = len(_markdown_table_cells(line)) + while index < len(lines): + candidate = lines[index] + established = ( + len(rows) >= 2 + and bool(_MARKDOWN_TABLE_SEPARATOR.match(rows[1])) + and len(_markdown_table_cells(rows[1])) == column_count + ) + if not _is_markdown_table_row(candidate) and not ( + established + and _is_empty_markdown_table_row(candidate, column_count) + ): + break + rows.append(candidate) index += 1 data_rows = [row for row in rows if not _MARKDOWN_TABLE_SEPARATOR.match(row)] if len(data_rows) >= 2: diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 8d1c457c6..ab70048b3 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -105,11 +105,62 @@ def test_chunk_by_dom_keeps_nested_table_cell_blocks_in_their_row() -> None: assert [(chunk.label, chunk.text) for chunk in chunks] == [("tr", "No. | Company")] -def test_chunk_by_dom_labels_markerless_footnotes() -> None: - chunks = chunk_by_dom("

Body text

*Tier 2: follow-up note

") +def test_chunk_by_dom_preserves_empty_table_cells() -> None: + chunks = chunk_by_dom( + "
Synthetic item
" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "| Synthetic item |") + ] + + +def test_chunk_by_dom_preserves_self_closing_empty_table_cells() -> None: + html_chunks = chunk_by_dom( + "
LeftRight
" + ) + word_chunks = chunk_by_dom( + "LeftRight" + ) + + assert [(chunk.label, chunk.text) for chunk in html_chunks] == [ + ("tr", "Left | | Right") + ] + assert [(chunk.label, chunk.text) for chunk in word_chunks] == [ + ("w:tr", "Left | | Right") + ] + + +def test_chunk_by_dom_scopes_cell_positions_to_nested_table_rows() -> None: + chunks = chunk_by_dom( + "
Outer left" + "
Inner leftInner right
" + "		Outer right
" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Inner left | Inner right"), + ("tr", "Outer left | Outer right"), + ] + + +def test_chunk_by_dom_implicitly_closes_sibling_rows_at_the_same_table_depth() -> None: + chunks = chunk_by_dom( + "" + "
First leftFirst right
Second leftSecond right
" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "First left | First right"), + ("tr", "Second left | Second right"), + ] + + +def test_chunk_by_dom_does_not_infer_a_footnote_from_a_bare_marker() -> None: + chunks = chunk_by_dom("

Body text

*Synthetic list item

") assert [(chunk.label, chunk.text) for chunk in chunks] == [ ("p", "Body text"), - ("footnote", "*Tier 2: follow-up note"), + ("p", "*Synthetic list item"), ] @@ -233,6 +284,38 @@ def test_chunk_by_source_body_splits_plain_lists_and_markdown_tables() -> None: ] +def test_chunk_by_source_body_preserves_empty_markdown_table_cells() -> None: + chunks = chunk_by_source_body( + "| Key | Value | State |\n" + "| --- | --- | --- |\n" + "| A | | Open |" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Key | Value | State"), + ("tr", "A | | Open"), + ] + + +def test_chunk_by_source_body_keeps_contextual_all_empty_markdown_rows() -> None: + chunks = chunk_by_source_body( + "| Key | Value | State |\n" + "| --- | --- | --- |\n" + "| | | |" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Key | Value | State"), + ("tr", "| |"), + ] + + +def test_chunk_by_source_body_does_not_promote_a_standalone_empty_pipe_line() -> None: + chunks = chunk_by_source_body("| | |") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [("", "| | |")] + + def test_chunk_by_dom_joins_visual_continuation_lines_but_keeps_list_items() -> None: html = ( '

1. 배경
' @@ -421,12 +504,28 @@ def test_normalize_script_text_maps_quantity_exponents_and_leaves_comparisons() assert normalize_script_text("^1 See the tank note.") == "^1 See the tank note." +def test_normalize_script_text_keeps_mixed_script_content_as_a_visible_fallback() -> None: + assert normalize_script_text("x3a") == "x^3a" + + +def test_normalize_script_text_decodes_nested_inline_markup_before_stripping() -> None: + assert normalize_script_text("x<span>2</span>") == "x²" + + def test_chunk_by_dom_keeps_html_quantity_scripts_as_unicode() -> None: chunks = chunk_by_dom("

Tank volume is 12 m3 of H2O.

") assert [chunk.text for chunk in chunks] == ["Tank volume is 12 m³ of H₂O."] +def test_chunk_by_dom_normalizes_entity_encoded_quantity_scripts() -> None: + chunks = chunk_by_dom( + "

Reserve 12 m^3 and x<sup>2</sup> units.

" + ) + + assert [chunk.text for chunk in chunks] == ["Reserve 12 m³ and x² units."] + + def test_chunk_by_dom_unclosed_sup_does_not_cross_table_cells() -> None: chunks = chunk_by_dom("
m3Acme Corp
")