diff --git a/frontend/src/PostBody.test.tsx b/frontend/src/PostBody.test.tsx index 76837b5b2..3eaba4970 100644 --- a/frontend/src/PostBody.test.tsx +++ b/frontend/src/PostBody.test.tsx @@ -58,6 +58,101 @@ describe("PostBody", () => { expect(screen.getByText("Embedded image")).toBeInTheDocument(); }); + it("renders raw and persisted encoded non-script markup as the same inert text", () => { + const encoded = + "Keep <b>bold</b>, <sup-note>2</sup-note>, " + + "<sub:item>3</sub:item>, and <script>alert(1)</script> literal."; + const visible = + "Keep bold, 2, 3, and literal."; + const { container, rerender } = render(${encoded}

`} />); + + expect(screen.getByText(visible)).toBeInTheDocument(); + expect(container.querySelector("b")).toBeNull(); + expect(container.querySelector("script")).toBeNull(); + expect(container.querySelector("sup-note")).toBeNull(); + + rerender( + ${encoded}

`} + structureUnits={[ + { + unit_index: 0, + unit_kind_code: "plain_text", + unit_text: encoded, + indent_level: 0, + indent_source_code: "explicit", + indent_confidence: 1, + indent_evidence: "Synthetic encoded source", + }, + ]} + />, + ); + + expect(screen.getByText(visible)).toBeInTheDocument(); + expect(container.querySelector("b")).toBeNull(); + expect(container.querySelector("script")).toBeNull(); + expect(container.querySelector("sup-note")).toBeNull(); + }); + + it("renders raw and legacy persisted encoded scripts with the same semantics", () => { + const encoded = + "Volume x<sup>2</sup>, coolant H<sub>2</sub>O, and area m&#94;3."; + const { container, rerender } = render(${encoded}

`} />); + + expect([...container.querySelectorAll("sup")].map((node) => node.textContent)).toEqual([ + "2", + "3", + ]); + expect(container.querySelector("sub")?.textContent).toBe("2"); + + rerender( + ${encoded}

`} + structureUnits={[ + { + unit_index: 0, + unit_kind_code: "plain_text", + unit_text: encoded, + indent_level: 0, + indent_source_code: "explicit", + indent_confidence: 1, + indent_evidence: "Synthetic legacy persisted unit", + }, + ]} + />, + ); + + expect([...container.querySelectorAll("sup")].map((node) => node.textContent)).toEqual([ + "2", + "3", + ]); + expect(container.querySelector("sub")?.textContent).toBe("2"); + }); + + it("normalizes legacy encoded scripts in persisted table cells", () => { + const { container } = render( + , + ); + + const superscript = container.querySelector("td sup"); + expect(superscript?.textContent).toBe("3"); + expect(superscript?.closest("td")?.textContent).toBe("12 m3"); + }); + it("renders authoritative LLM structure levels for semantic list units", () => { render( ( {row.unit_text.split(/\s*\|\s*/).map((cell, cellIndex) => ( - {renderStyledText(cell)} + + {renderStyledText(displayUnitText(cell))} + ))} ))} @@ -251,7 +263,7 @@ function renderStructuredUnits( renderSegment( { kind: "text", - text: unit.unit_text, + text: displayUnitText(unit.unit_text), ...(unit.unit_label === "footnote" || sourceText?.role === "footnote" ? { role: "footnote" as const } : {}), diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index 20e266122..3a5a5e3af 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -166,6 +166,22 @@ describe("splitPostBody", () => { ).toEqual([{ kind: "text", text: "Reserve 12 m³ and x² units." }]); }); + it("keeps invalid encoded script pairs literal", () => { + expect( + splitPostBody( + "

Keep x<sup>2 unmatched; x<sup/>2 self-closing; " + + "x<sup class="unit">2</sup> attributed; and " + + "x<sup>2</sub> mismatched.

", + ), + ).toEqual([ + { + kind: "text", + text: + 'Keep x2 unmatched; x2 self-closing; x2 attributed; and x2 mismatched.', + }, + ]); + }); + it("keeps encoded non-script inline markup literal", () => { expect(splitPostBody("

Keep <b>bold</b> literal.

")).toEqual([ { kind: "text", text: "Keep bold literal." }, diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index f037a8f1a..6a4f17540 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -39,8 +39,10 @@ export function decodeHtmlEntities(text: string): string { const decoder = document.createElement("textarea"); let decoded = text; for (let pass = 0; pass < 3; pass += 1) { - decoder.innerHTML = decoded; - const next = decoder.value; + const next = decoded.replace(/&(?:#[0-9]+|#x[0-9a-f]+|[a-z][a-z0-9]+);/gi, (entity) => { + decoder.innerHTML = entity; + return decoder.value; + }); if (next === decoded) break; decoded = next; } @@ -156,8 +158,15 @@ const SUB_UNI_TO_ASCII = buildUnicodeToAsciiTable(SUB_ASCII_TO_UNI); const CARET_EXPONENT = /(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+-]?\d{1,3}|[nNiI])\}|([+-]?\d{1,3}|[nNiI]))/g; const ENCODED_CARET = /&(?:amp;)*(?:#0*94|#x0*5e);/gi; -const ENCODED_SCRIPT_TAG = - /&(?:amp;)*(?:lt|#0*60|#x0*3c);\s*\/?\s*(?:sup|sub)(?=\s|\/|&(?:amp;)*(?:gt|#0*62|#x0*3e);).*?&(?:amp;)*(?:gt|#0*62|#x0*3e);/gis; +const ENCODED_LT = String.raw`&(?:amp;)*(?:lt|#0*60|#x0*3c);`; +const ENCODED_GT = String.raw`&(?:amp;)*(?:gt|#0*62|#x0*3e);`; +const ENCODED_SCRIPT_TOKEN = + `${ENCODED_LT}\\s*/?\\s*(?:sup|sub)(?=\\s|/|${ENCODED_GT})`; +const ENCODED_SCRIPT_PAIR = new RegExp( + `${ENCODED_LT}(sup|sub)${ENCODED_GT}` + + `((?:(?!${ENCODED_SCRIPT_TOKEN}).)*?)${ENCODED_LT}/\\1${ENCODED_GT}`, + "gis", +); function applyUnicodeScript(text: string, kind: "super" | "sub"): string { const table = kind === "super" ? SUPER_ASCII_TO_UNI : SUB_ASCII_TO_UNI; @@ -185,7 +194,11 @@ function replaceHtmlScripts(text: string): string { function decodeScriptEntities(text: string): string { return text - .replace(ENCODED_SCRIPT_TAG, (tag) => decodeHtmlEntities(tag)) + .replace( + ENCODED_SCRIPT_PAIR, + (_pair, kind: string, inner: string) => + `<${kind.toLowerCase()}>${inner}`, + ) .replace(ENCODED_CARET, (caret) => decodeHtmlEntities(caret)); } diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 511634ff7..fc66e2ec3 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -194,6 +194,18 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool: _CARET_EXPONENT = re.compile( r"(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+\-]?\d{1,3}|[nNiI])\}|([+\-]?\d{1,3}|[nNiI]))" ) +_ENCODED_CARET = re.compile(r"&(?:amp;)*(?:#0*94|#x0*5e);", re.IGNORECASE) +_ENCODED_LT = r"&(?:amp;)*(?:lt|#0*60|#x0*3c);" +_ENCODED_GT = r"&(?:amp;)*(?:gt|#0*62|#x0*3e);" +_ENCODED_SCRIPT_TOKEN = ( + rf"{_ENCODED_LT}\s*/?\s*(?:sup|sub)(?=\s|/|{_ENCODED_GT})" +) +_ENCODED_SCRIPT_PAIR = re.compile( + rf"{_ENCODED_LT}(?Psup|sub){_ENCODED_GT}" + rf"(?P(?:(?!{_ENCODED_SCRIPT_TOKEN}).)*?)" + rf"{_ENCODED_LT}/(?P=kind){_ENCODED_GT}", + re.IGNORECASE | re.DOTALL, +) def apply_unicode_script(text: str, kind: str) -> str: @@ -213,13 +225,30 @@ def apply_unicode_script(text: str, kind: str) -> str: return f"{leading}{prefix}{compact}{trailing}" -def _replace_html_script(match: re.Match[str], kind: str) -> str: - inner = match.group(1) +def _decode_html_entities(text: str) -> str: for _ in range(3): - decoded = unescape(inner) - if decoded == inner: + decoded = unescape(text) + if decoded == text: break - inner = decoded + text = decoded + return text + + +def _decode_script_entities(text: str) -> str: + decoded_pairs = _ENCODED_SCRIPT_PAIR.sub( + lambda match: ( + f"<{match.group('kind').lower()}>{match.group('inner')}" + f"" + ), + text, + ) + return _ENCODED_CARET.sub( + lambda match: _decode_html_entities(match.group(0)), decoded_pairs + ) + + +def _replace_html_script(match: re.Match[str], kind: str) -> str: + inner = _decode_html_entities(match.group(1)) return apply_unicode_script(_INNER_TAG.sub("", inner), kind) @@ -227,7 +256,7 @@ def normalize_script_text(text: str) -> str: """Turn HTML/caret quantity scripts into Unicode without treating comparisons as tags.""" replaced = _CARET_EXPONENT.sub( lambda match: apply_unicode_script(match.group(1) or match.group(2), "sup"), - text, + _decode_script_entities(text), ) replaced = _HTML_SUP.sub(lambda match: _replace_html_script(match, "sup"), replaced) replaced = _HTML_SUB.sub(lambda match: _replace_html_script(match, "sub"), replaced) diff --git a/tests/test_chunking.py b/tests/test_chunking.py index ab70048b3..21cf75b05 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -284,6 +284,38 @@ def test_chunk_by_source_body_splits_plain_lists_and_markdown_tables() -> None: ] +def test_chunk_by_source_body_normalizes_entity_encoded_quantity_scripts() -> None: + chunks = chunk_by_source_body( + "Reserve 12 m^3, x<sup>2</sup>, and H<sub>2</sub>O." + ) + + assert [chunk.text for chunk in chunks] == ["Reserve 12 m³, x², and H₂O."] + + +def test_chunk_by_source_body_keeps_invalid_encoded_script_pairs_literal() -> None: + bodies = ( + "Keep x<sup>2 unmatched.", + "Keep x<sup/>2 self-closing.", + "Keep x<sup class="unit">2</sup> attributed.", + "Keep x<sup>2</sub> mismatched.", + ) + + assert [chunk_by_source_body(body)[0].text for body in bodies] == list(bodies) + combined = " ".join(bodies) + assert chunk_by_source_body(combined)[0].text == combined + + +def test_chunk_by_source_body_keeps_encoded_non_script_markup_inert() -> None: + body = ( + "Keep <b>bold</b>, <sup-note>2</sup-note>, " + "and <script>alert(1)</script> literal." + ) + + chunks = chunk_by_source_body(body) + + assert [(chunk.unit_type, chunk.text) for chunk in chunks] == [("plain_text", body)] + + def test_chunk_by_source_body_preserves_empty_markdown_table_cells() -> None: chunks = chunk_by_source_body( "| Key | Value | State |\n"