fix: preserve encoded-script source parity

frontend/src/PostBody.test.tsx

@@ -58,6 +58,101 @@ describe("PostBody", () => {
     expect(screen.getByText("Embedded image")).toBeInTheDocument();
   });
 
+  it("renders raw and persisted encoded non-script markup as the same inert text", () => {
+    const encoded =
+      "Keep <b>bold</b>, <sup-note>2</sup-note>, " +
+      "<sub:item>3</sub:item>, and <script>alert(1)</script> literal.";
+    const visible =
+      "Keep <b>bold</b>, <sup-note>2</sup-note>, <sub:item>3</sub:item>, and <script>alert(1)</script> literal.";
+    const { container, rerender } = render(<PostBody body={`<p>${encoded}</p>`} />);
+
+    expect(screen.getByText(visible)).toBeInTheDocument();
+    expect(container.querySelector("b")).toBeNull();
+    expect(container.querySelector("script")).toBeNull();
+    expect(container.querySelector("sup-note")).toBeNull();
+
+    rerender(
+      <PostBody
+        body={`<p>${encoded}</p>`}
+        structureUnits={[
+          {
+            unit_index: 0,
+            unit_kind_code: "plain_text",
+            unit_text: encoded,
+            indent_level: 0,
+            indent_source_code: "explicit",
+            indent_confidence: 1,
+            indent_evidence: "Synthetic encoded source",
+          },
+        ]}
+      />,
+    );
+
+    expect(screen.getByText(visible)).toBeInTheDocument();
+    expect(container.querySelector("b")).toBeNull();
+    expect(container.querySelector("script")).toBeNull();
+    expect(container.querySelector("sup-note")).toBeNull();
+  });
+
+  it("renders raw and legacy persisted encoded scripts with the same semantics", () => {
+    const encoded =
+      "Volume x&lt;sup&gt;2&lt;/sup&gt;, coolant H&lt;sub&gt;2&lt;/sub&gt;O, and area m&amp;#94;3.";
+    const { container, rerender } = render(<PostBody body={`<p>${encoded}</p>`} />);
+
+    expect([...container.querySelectorAll("sup")].map((node) => node.textContent)).toEqual([
+      "2",
+      "3",
+    ]);
+    expect(container.querySelector("sub")?.textContent).toBe("2");
+
+    rerender(
+      <PostBody
+        body={`<p>${encoded}</p>`}
+        structureUnits={[
+          {
+            unit_index: 0,
+            unit_kind_code: "plain_text",
+            unit_text: encoded,
+            indent_level: 0,
+            indent_source_code: "explicit",
+            indent_confidence: 1,
+            indent_evidence: "Synthetic legacy persisted unit",
+          },
+        ]}
+      />,
+    );
+
+    expect([...container.querySelectorAll("sup")].map((node) => node.textContent)).toEqual([
+      "2",
+      "3",
+    ]);
+    expect(container.querySelector("sub")?.textContent).toBe("2");
+  });
+
+  it("normalizes legacy encoded scripts in persisted table cells", () => {
+    const { container } = render(
+      <PostBody
+        body="<table><tr><td>Measure</td><td>Volume</td></tr></table>"
+        structureUnits={[
+          {
+            unit_index: 0,
+            unit_kind_code: "table_row",
+            unit_label: "tr",
+            unit_text: "Measure | 12 m&lt;sup&gt;3&lt;/sup&gt;",
+            indent_level: 0,
+            indent_source_code: "explicit",
+            indent_confidence: 1,
+            indent_evidence: "Synthetic legacy persisted table row",
+          },
+        ]}
+      />,
+    );
+
+    const superscript = container.querySelector("td sup");
+    expect(superscript?.textContent).toBe("3");
+    expect(superscript?.closest("td")?.textContent).toBe("12 m3");
+  });
+
   it("renders authoritative LLM structure levels for semantic list units", () => {
     render(
       <PostBody

frontend/src/PostBody.tsx

@@ -1,4 +1,10 @@
-import { splitPostBody, splitScriptRuns, normalizeScriptText, type PostBodySegment } from "./postBodyDisplay";
+import {
+  decodeHtmlEntities,
+  splitPostBody,
+  splitScriptRuns,
+  normalizeScriptText,
+  type PostBodySegment,
+} from "./postBodyDisplay";
 import { t } from "./i18n";
 import type { PostContentUnit, PostImageContent } from "./api";
 import { Fragment, type ReactNode } from "react";
@@ -140,8 +146,12 @@ function isStructuredTableRow(unit: PostContentUnit): boolean {
  * its source display is still one text segment, so ordinal matching shifts
  * indentation for every later unresolved unit.
  */
+function displayUnitText(value: string): string {
+  return decodeHtmlEntities(normalizeScriptText(value));
+}
+
 function normalizedUnitText(value: string): string {
-  return normalizeScriptText(value.replace(/\s+/g, " ").trim());
+  return displayUnitText(value).replace(/\s+/g, " ").trim();
 }
 
 /**
@@ -232,7 +242,9 @@ function renderStructuredUnits(
             {rows.map((row, rowIndex) => (
               <tr key={`post-body-table-row-${row.unit_index}-${rowIndex}`}>
                 {row.unit_text.split(/\s*\|\s*/).map((cell, cellIndex) => (
-                  <td key={`post-body-table-cell-${row.unit_index}-${cellIndex}`}>{renderStyledText(cell)}</td>
+                  <td key={`post-body-table-cell-${row.unit_index}-${cellIndex}`}>
+                    {renderStyledText(displayUnitText(cell))}
+                  </td>
                 ))}
               </tr>
             ))}
@@ -251,7 +263,7 @@ function renderStructuredUnits(
       renderSegment(
         {
           kind: "text",
-          text: unit.unit_text,
+          text: displayUnitText(unit.unit_text),
           ...(unit.unit_label === "footnote" || sourceText?.role === "footnote"
             ? { role: "footnote" as const }
             : {}),

frontend/src/postBodyDisplay.test.ts

@@ -166,6 +166,22 @@ describe("splitPostBody", () => {
     ).toEqual([{ kind: "text", text: "Reserve 12 m³ and x² units." }]);
   });
 
+  it("keeps invalid encoded script pairs literal", () => {
+    expect(
+      splitPostBody(
+        "<p>Keep x&lt;sup&gt;2 unmatched; x&lt;sup/&gt;2 self-closing; " +
+          "x&lt;sup class=&quot;unit&quot;&gt;2&lt;/sup&gt; attributed; and " +
+          "x&lt;sup&gt;2&lt;/sub&gt; mismatched.</p>",
+      ),
+    ).toEqual([
+      {
+        kind: "text",
+        text:
+          'Keep x<sup>2 unmatched; x<sup/>2 self-closing; x<sup class="unit">2</sup> attributed; and x<sup>2</sub> mismatched.',
+      },
+    ]);
+  });
+
   it("keeps encoded non-script inline markup literal", () => {
     expect(splitPostBody("<p>Keep &lt;b&gt;bold&lt;/b&gt; literal.</p>")).toEqual([
       { kind: "text", text: "Keep <b>bold</b> literal." },

frontend/src/postBodyDisplay.ts

@@ -39,8 +39,10 @@ export function decodeHtmlEntities(text: string): string {
   const decoder = document.createElement("textarea");
   let decoded = text;
   for (let pass = 0; pass < 3; pass += 1) {
-    decoder.innerHTML = decoded;
-    const next = decoder.value;
+    const next = decoded.replace(/&(?:#[0-9]+|#x[0-9a-f]+|[a-z][a-z0-9]+);/gi, (entity) => {
+      decoder.innerHTML = entity;
+      return decoder.value;
+    });
     if (next === decoded) break;
     decoded = next;
   }
@@ -156,8 +158,15 @@ const SUB_UNI_TO_ASCII = buildUnicodeToAsciiTable(SUB_ASCII_TO_UNI);
 const CARET_EXPONENT =
   /(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+-]?\d{1,3}|[nNiI])\}|([+-]?\d{1,3}|[nNiI]))/g;
 const ENCODED_CARET = /&(?:amp;)*(?:#0*94|#x0*5e);/gi;
-const ENCODED_SCRIPT_TAG =
-  /&(?:amp;)*(?:lt|#0*60|#x0*3c);\s*\/?\s*(?:sup|sub)(?=\s|\/|&(?:amp;)*(?:gt|#0*62|#x0*3e);).*?&(?:amp;)*(?:gt|#0*62|#x0*3e);/gis;
+const ENCODED_LT = String.raw`&(?:amp;)*(?:lt|#0*60|#x0*3c);`;
+const ENCODED_GT = String.raw`&(?:amp;)*(?:gt|#0*62|#x0*3e);`;
+const ENCODED_SCRIPT_TOKEN =
+  `${ENCODED_LT}\\s*/?\\s*(?:sup|sub)(?=\\s|/|${ENCODED_GT})`;
+const ENCODED_SCRIPT_PAIR = new RegExp(
+  `${ENCODED_LT}(sup|sub)${ENCODED_GT}` +
+    `((?:(?!${ENCODED_SCRIPT_TOKEN}).)*?)${ENCODED_LT}/\\1${ENCODED_GT}`,
+  "gis",
+);
 
 function applyUnicodeScript(text: string, kind: "super" | "sub"): string {
   const table = kind === "super" ? SUPER_ASCII_TO_UNI : SUB_ASCII_TO_UNI;
@@ -185,7 +194,11 @@ function replaceHtmlScripts(text: string): string {
 
 function decodeScriptEntities(text: string): string {
   return text
-    .replace(ENCODED_SCRIPT_TAG, (tag) => decodeHtmlEntities(tag))
+    .replace(
+      ENCODED_SCRIPT_PAIR,
+      (_pair, kind: string, inner: string) =>
+        `<${kind.toLowerCase()}>${inner}</${kind.toLowerCase()}>`,
+    )
     .replace(ENCODED_CARET, (caret) => decodeHtmlEntities(caret));
 }
 

lineageweave/chunking.py

@@ -194,6 +194,18 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool:
 _CARET_EXPONENT = re.compile(
     r"(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+\-]?\d{1,3}|[nNiI])\}|([+\-]?\d{1,3}|[nNiI]))"
 )
+_ENCODED_CARET = re.compile(r"&(?:amp;)*(?:#0*94|#x0*5e);", re.IGNORECASE)
+_ENCODED_LT = r"&(?:amp;)*(?:lt|#0*60|#x0*3c);"
+_ENCODED_GT = r"&(?:amp;)*(?:gt|#0*62|#x0*3e);"
+_ENCODED_SCRIPT_TOKEN = (
+    rf"{_ENCODED_LT}\s*/?\s*(?:sup|sub)(?=\s|/|{_ENCODED_GT})"
+)
+_ENCODED_SCRIPT_PAIR = re.compile(
+    rf"{_ENCODED_LT}(?P<kind>sup|sub){_ENCODED_GT}"
+    rf"(?P<inner>(?:(?!{_ENCODED_SCRIPT_TOKEN}).)*?)"
+    rf"{_ENCODED_LT}/(?P=kind){_ENCODED_GT}",
+    re.IGNORECASE | re.DOTALL,
+)
 
 
 def apply_unicode_script(text: str, kind: str) -> str:
@@ -213,21 +225,38 @@ def apply_unicode_script(text: str, kind: str) -> str:
     return f"{leading}{prefix}{compact}{trailing}"
 
 
-def _replace_html_script(match: re.Match[str], kind: str) -> str:
-    inner = match.group(1)
+def _decode_html_entities(text: str) -> str:
     for _ in range(3):
-        decoded = unescape(inner)
-        if decoded == inner:
+        decoded = unescape(text)
+        if decoded == text:
             break
-        inner = decoded
+        text = decoded
+    return text
+
+
+def _decode_script_entities(text: str) -> str:
+    decoded_pairs = _ENCODED_SCRIPT_PAIR.sub(
+        lambda match: (
+            f"<{match.group('kind').lower()}>{match.group('inner')}"
+            f"</{match.group('kind').lower()}>"
+        ),
+        text,
+    )
+    return _ENCODED_CARET.sub(
+        lambda match: _decode_html_entities(match.group(0)), decoded_pairs
+    )
+
+
+def _replace_html_script(match: re.Match[str], kind: str) -> str:
+    inner = _decode_html_entities(match.group(1))
     return apply_unicode_script(_INNER_TAG.sub("", inner), kind)
 
 
 def normalize_script_text(text: str) -> str:
     """Turn HTML/caret quantity scripts into Unicode without treating comparisons as tags."""
     replaced = _CARET_EXPONENT.sub(
         lambda match: apply_unicode_script(match.group(1) or match.group(2), "sup"),
-        text,
+        _decode_script_entities(text),
     )
     replaced = _HTML_SUP.sub(lambda match: _replace_html_script(match, "sup"), replaced)
     replaced = _HTML_SUB.sub(lambda match: _replace_html_script(match, "sub"), replaced)

tests/test_chunking.py

@@ -284,6 +284,38 @@ def test_chunk_by_source_body_splits_plain_lists_and_markdown_tables() -> None:
     ]
 
 
+def test_chunk_by_source_body_normalizes_entity_encoded_quantity_scripts() -> None:
+    chunks = chunk_by_source_body(
+        "Reserve 12 m^3, x<sup>2</sup>, and H<sub>2</sub>O."
+    )
+
+    assert [chunk.text for chunk in chunks] == ["Reserve 12 m³, x², and H₂O."]
+
+
+def test_chunk_by_source_body_keeps_invalid_encoded_script_pairs_literal() -> None:
+    bodies = (
+        "Keep x<sup>2 unmatched.",
+        "Keep x<sup/>2 self-closing.",
+        "Keep x<sup class="unit">2</sup> attributed.",
+        "Keep x<sup>2</sub> mismatched.",
+    )
+
+    assert [chunk_by_source_body(body)[0].text for body in bodies] == list(bodies)
+    combined = " ".join(bodies)
+    assert chunk_by_source_body(combined)[0].text == combined
+
+
+def test_chunk_by_source_body_keeps_encoded_non_script_markup_inert() -> None:
+    body = (
+        "Keep <b>bold</b>, <sup-note>2</sup-note>, "
+        "and <script>alert(1)</script> literal."
+    )
+
+    chunks = chunk_by_source_body(body)
+
+    assert [(chunk.unit_type, chunk.text) for chunk in chunks] == [("plain_text", body)]
+
+
 def test_chunk_by_source_body_preserves_empty_markdown_table_cells() -> None:
     chunks = chunk_by_source_body(
         "| Key | Value | State |\n"