From 11a60b370d7b5783733febb593e8f91678cc403d Mon Sep 17 00:00:00 2001
From: Seongho Bae
Date: Sat, 22 Aug 2026 18:18:22 +0900
Subject: [PATCH 01/48] fix(frontend): repair the inherited login/admin-panel
build break
Two TypeScript build errors on main (blocking every open PR's
"Frontend lint, test, build" check, including this repo's own review
bot's ability to approve them):
- App.tsx imported rememberOidcReturnUrl/returnUrlFromLocation from
oidcReturnUrl.ts but never called them -- the login button built its
own unsanitized returnUrl inline instead of using the safe helper
(oidcReturnUrl.ts's isSafeReturnUrl guard against an open-redirect-
shaped value) or persisting it as the sessionStorage/localStorage
fallback restoreOidcReturnUrl (already wired up on the callback side
in main.tsx) reads when the OIDC state round-trip drops it.
- The unauthenticated login screen unconditionally rendered
when destination === "admin"
-- accessToken is string | undefined here (always undefined while
unauthenticated), a real type error, and the render was unreachable
through normal navigation (destination only changes via the
authenticated nav) -- dead code, removed.
uv run --frozen python -m pytest -q: 753 passed, 17 skipped.
pnpm run test: 140 passed. pnpm run lint / build: clean.
---
frontend/src/App.test.tsx | 3 +++
frontend/src/App.tsx | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx
index 7462abd2c..70eb27590 100644
--- a/frontend/src/App.test.tsx
+++ b/frontend/src/App.test.tsx
@@ -41,6 +41,9 @@ describe("App, unauthenticated", () => {
state: expect.objectContaining({ returnUrl: expect.stringMatching(/^\//) }),
}),
);
+ // Persisted as a fallback in case the OIDC state round-trip is dropped
+ // (see oidcReturnUrl.ts's restoreOidcReturnUrl, consumed in main.tsx).
+ expect(window.sessionStorage.getItem("lineageweave.oidc.returnUrl")).toMatch(/^\//);
});
});
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 6fba0dd41..1b5b351ab 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -4610,7 +4610,8 @@ export default function App({ showLabPanels = false }: { showLabPanels?: boolean
',
),
).toEqual([
{ kind: "text", text: "Outer", indentLevel: 7 },
{ kind: "text", text: "Nested", indentLevel: 10 },
- { kind: "text", text: "*Tier 2: note", role: "footnote" },
+ ]);
+ });
+
+ it("does not infer a footnote from a bare marker", () => {
+ expect(splitPostBody("
*Synthetic list item
")).toEqual([
+ { kind: "text", text: "*Synthetic list item" },
]);
});
@@ -132,4 +136,139 @@ describe("splitPostBody", () => {
);
expect(JSON.stringify(segments)).not.toContain("https://example.test");
});
+
+ it("turns HTML and caret quantity exponents into unicode without flattening them", () => {
+ expect(splitPostBody("
Tank volume is 12 m3.
")).toEqual([
+ { kind: "text", text: "Tank volume is 12 m³." },
+ ]);
+ expect(splitPostBody("Tank volume is 12 m^3.")).toEqual([
+ { kind: "text", text: "Tank volume is 12 m³." },
+ ]);
+ expect(splitPostBody("Coolant is H2O at 10^{-3} M.")).toEqual([
+ { kind: "text", text: "Coolant is H₂O at 10⁻³ M." },
+ ]);
+ });
+
+ it("decodes HTML entities inside a sup/sub tag before mapping to unicode", () => {
+ // Office-tool HTML export pads content with . The raw,
+ // un-decoded " 3" must not fail the all-convertible check and fall
+ // back to a literal caret (regression: entities weren't decoded before
+ // the convertibility check, matching the Python backend which unescapes
+ // first).
+ expect(splitPostBody("
Volume is 12 m 3.
")).toEqual([
+ { kind: "text", text: "Volume is 12 m ³." },
+ ]);
+ });
+
+ it("normalizes entity-encoded quantity syntax without leaking raw markup", () => {
+ expect(
+ splitPostBody("
Keep <sup-note>2</sup-note> and <sub:item>3</sub:item> literal.
",
+ ),
+ ).toEqual([
+ {
+ kind: "text",
+ text: "Keep 2 and 3 literal.",
+ },
+ ]);
+ });
+
+ it("matches sup/sub content split across a newline", () => {
+ // Pretty-printed source HTML puts tag content on its own line
+ // (regression: the regex lacked the dotAll flag, so `.` could not cross
+ // the newline and the whole tag passed through unmatched, leaving a
+ // plain un-superscripted "3" instead of "³").
+ expect(splitPostBody("
Tank volume is 12 m\n3\n.
")).toEqual([
+ { kind: "text", text: "Tank volume is 12 m ³ ." },
+ ]);
+ });
+
+ it("does not treat a leading footnote caret or a comparison as an exponent", () => {
+ expect(splitPostBody("^1 See the tank note.")).toEqual([
+ { kind: "text", text: "^1 See the tank note." },
+ ]);
+ expect(normalizeScriptText("qty < 50 and price > 10")).toBe("qty < 50 and price > 10");
+ expect(splitScriptRuns("Tank volume is 12 m³.")).toEqual([
+ { text: "Tank volume is 12 m" },
+ { text: "3", script: "super" },
+ { text: "." },
+ ]);
+ });
+
+ it("keeps mixed script content as a visible fallback", () => {
+ expect(splitPostBody("x3a")).toEqual([{ kind: "text", text: "x^3a" }]);
+ });
+
+ it("decodes a stored superscript letter deterministically to lowercase", () => {
+ // "n" and "N" both encode to the same Unicode "ⁿ" (there is no distinct
+ // uppercase superscript N), so decoding must pick one case consistently
+ // rather than depending on object key iteration order (regression: used
+ // to always decode to uppercase because "N"/"I" were inserted after
+ // "n"/"i" in the forward table).
+ expect(splitScriptRuns("mⁿ")).toEqual([
+ { text: "m" },
+ { text: "n", script: "super" },
+ ]);
+ expect(splitScriptRuns("xⁱ")).toEqual([
+ { text: "x" },
+ { text: "i", script: "super" },
+ ]);
+ expect(splitPostBody("mN")).toEqual([{ kind: "text", text: "mⁿ" }]);
+ expect(splitScriptRuns(normalizeScriptText("mN"))).toEqual([
+ { text: "m" },
+ { text: "n", script: "super" },
+ ]);
+ });
});
diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts
index 919e8c0ca..6a4f17540 100644
--- a/frontend/src/postBodyDisplay.ts
+++ b/frontend/src/postBodyDisplay.ts
@@ -20,7 +20,6 @@ const BLOCK_TAG =
/<\/?(?:article|blockquote|div|h[1-6]|li|ol|p|section|table|tbody|td|tfoot|th|thead|tr|ul|w:p|w:tbl|w:tr|w:tc)\b[^>]*>/gi;
const WORD_INDENT_TAG = /]*\/?\s*>/gi;
const LIST_ITEM_START = /^\s*(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)/;
-const FOOTNOTE_START = /^\s*[*†‡](?=\S)/;
const INDENT_MARKER = "\u0001lw-indent:";
const INDENT_MARKER_END = "\u0002";
const INDENT_MARKER_PATTERN = /lw-indent:(\d+)/g;
@@ -40,8 +39,10 @@ export function decodeHtmlEntities(text: string): string {
const decoder = document.createElement("textarea");
let decoded = text;
for (let pass = 0; pass < 3; pass += 1) {
- decoder.innerHTML = decoded;
- const next = decoder.value;
+ const next = decoded.replace(/&(?:#[0-9]+|#x[0-9a-f]+|[a-z][a-z0-9]+);/gi, (entity) => {
+ decoder.innerHTML = entity;
+ return decoder.value;
+ });
if (next === decoded) break;
decoded = next;
}
@@ -89,9 +90,180 @@ function indentMarker(width: number): string {
return width > 0 ? `${INDENT_MARKER}${width}${INDENT_MARKER_END}` : "";
}
+const SUPER_ASCII_TO_UNI: Record = {
+ "0": "⁰",
+ "1": "¹",
+ "2": "²",
+ "3": "³",
+ "4": "⁴",
+ "5": "⁵",
+ "6": "⁶",
+ "7": "⁷",
+ "8": "⁸",
+ "9": "⁹",
+ "+": "⁺",
+ "-": "⁻",
+ "=": "⁼",
+ "(": "⁽",
+ ")": "⁾",
+ n: "ⁿ",
+ N: "ⁿ",
+ i: "ⁱ",
+ I: "ⁱ",
+};
+const SUB_ASCII_TO_UNI: Record = {
+ "0": "₀",
+ "1": "₁",
+ "2": "₂",
+ "3": "₃",
+ "4": "₄",
+ "5": "₅",
+ "6": "₆",
+ "7": "₇",
+ "8": "₈",
+ "9": "₉",
+ "+": "₊",
+ "-": "₋",
+ "=": "₌",
+ "(": "₍",
+ ")": "₎",
+ a: "ₐ",
+ e: "ₑ",
+ h: "ₕ",
+ i: "ᵢ",
+ k: "ₖ",
+ l: "ₗ",
+ m: "ₘ",
+ n: "ₙ",
+ o: "ₒ",
+ p: "ₚ",
+ s: "ₛ",
+ t: "ₜ",
+ x: "ₓ",
+};
+// Two ASCII keys can map to the same Unicode character (e.g. "n" and "N"
+// both produce "ⁿ"). Building the reverse table naively lets the
+// last-inserted ASCII key win, so decoding always yields one fixed case
+// regardless of what was actually stored. Keep the first (lowercase, since
+// it is listed first above) mapping instead, so round-tripping preserves case.
+function buildUnicodeToAsciiTable(table: Record): Record {
+ const reverse: Record = {};
+ for (const [ascii, uni] of Object.entries(table)) {
+ if (!(uni in reverse)) reverse[uni] = ascii;
+ }
+ return reverse;
+}
+const SUPER_UNI_TO_ASCII = buildUnicodeToAsciiTable(SUPER_ASCII_TO_UNI);
+const SUB_UNI_TO_ASCII = buildUnicodeToAsciiTable(SUB_ASCII_TO_UNI);
+const CARET_EXPONENT =
+ /(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+-]?\d{1,3}|[nNiI])\}|([+-]?\d{1,3}|[nNiI]))/g;
+const ENCODED_CARET = /&(?:amp;)*(?:#0*94|#x0*5e);/gi;
+const ENCODED_LT = String.raw`&(?:amp;)*(?:lt|#0*60|#x0*3c);`;
+const ENCODED_GT = String.raw`&(?:amp;)*(?:gt|#0*62|#x0*3e);`;
+const ENCODED_SCRIPT_TOKEN =
+ `${ENCODED_LT}\\s*/?\\s*(?:sup|sub)(?=\\s|/|${ENCODED_GT})`;
+const ENCODED_SCRIPT_PAIR = new RegExp(
+ `${ENCODED_LT}(sup|sub)${ENCODED_GT}` +
+ `((?:(?!${ENCODED_SCRIPT_TOKEN}).)*?)${ENCODED_LT}/\\1${ENCODED_GT}`,
+ "gis",
+);
+
+function applyUnicodeScript(text: string, kind: "super" | "sub"): string {
+ const table = kind === "super" ? SUPER_ASCII_TO_UNI : SUB_ASCII_TO_UNI;
+ const values = new Set(Object.values(table));
+ const compact = text.trim();
+ if (!compact) return text;
+ if ([...compact].every((ch) => ch in table || values.has(ch) || /\s/.test(ch))) {
+ return [...text].map((ch) => table[ch] ?? ch).join("");
+ }
+ const prefix = kind === "super" ? "^" : "_";
+ const leading = text.match(/^\s*/)?.[0] ?? "";
+ const trailing = compact.length ? text.slice(leading.length + compact.length) : "";
+ return `${leading}${prefix}${compact}${trailing}`;
+}
+
+function replaceHtmlScripts(text: string): string {
+ return text
+ .replace(/]*>(.*?)<\/sup>/gis, (_match, inner: string) =>
+ applyUnicodeScript(decodeHtmlEntities(String(inner)).replace(/<[^>]+>/g, ""), "super"),
+ )
+ .replace(/]*>(.*?)<\/sub>/gis, (_match, inner: string) =>
+ applyUnicodeScript(decodeHtmlEntities(String(inner)).replace(/<[^>]+>/g, ""), "sub"),
+ );
+}
+
+function decodeScriptEntities(text: string): string {
+ return text
+ .replace(
+ ENCODED_SCRIPT_PAIR,
+ (_pair, kind: string, inner: string) =>
+ `<${kind.toLowerCase()}>${inner}${kind.toLowerCase()}>`,
+ )
+ .replace(ENCODED_CARET, (caret) => decodeHtmlEntities(caret));
+}
+
+export function normalizeScriptText(text: string): string {
+ const withCarets = decodeScriptEntities(text).replace(
+ CARET_EXPONENT,
+ (_match, braced: string, bare: string) => applyUnicodeScript(braced || bare, "super"),
+ );
+ return replaceHtmlScripts(withCarets);
+}
+
+export type ScriptRun = { text: string; script?: "super" | "sub" };
+
+export function splitScriptRuns(text: string): ScriptRun[] {
+ const runs: ScriptRun[] = [];
+ const push = (chunk: string, script?: "super" | "sub") => {
+ if (!chunk) return;
+ const last = runs[runs.length - 1];
+ if (last && last.script === script) {
+ last.text += chunk;
+ return;
+ }
+ runs.push(script ? { text: chunk, script } : { text: chunk });
+ };
+ let index = 0;
+ while (index < text.length) {
+ const ch = text[index];
+ if (ch in SUPER_UNI_TO_ASCII) {
+ let ascii = SUPER_UNI_TO_ASCII[ch];
+ index += 1;
+ while (index < text.length && text[index] in SUPER_UNI_TO_ASCII) {
+ ascii += SUPER_UNI_TO_ASCII[text[index]];
+ index += 1;
+ }
+ push(ascii, "super");
+ continue;
+ }
+ if (ch in SUB_UNI_TO_ASCII) {
+ let ascii = SUB_UNI_TO_ASCII[ch];
+ index += 1;
+ while (index < text.length && text[index] in SUB_UNI_TO_ASCII) {
+ ascii += SUB_UNI_TO_ASCII[text[index]];
+ index += 1;
+ }
+ push(ascii, "sub");
+ continue;
+ }
+ if (ch === "^" && index > 0 && /[A-Za-z0-9µμ°ΩÅåÅ)]/.test(text[index - 1])) {
+ const rest = text.slice(index);
+ const match = rest.match(/^\^(?:\{([+-]?\d{1,3}|[nNiI])\}|([+-]?\d{1,3}|[nNiI]))/);
+ if (match) {
+ push(match[1] || match[2] || "", "super");
+ index += match[0].length;
+ continue;
+ }
+ }
+ push(ch);
+ index += 1;
+ }
+ return runs;
+}
+
function stripHtmlTags(text: string): string {
- text = text.replace(/]*>(.*?)<\/sup>/gi, "^$1");
- const withBoundaries = text
+ const withScripts = normalizeScriptText(text);
+ const withBoundaries = withScripts
.replace(BREAK_TAG, "\n")
.replace(BLOCK_TAG, (tag) => {
if (/^<\//.test(tag)) return "\n\n";
@@ -101,8 +273,7 @@ function stripHtmlTags(text: string): string {
const withoutTags = withBoundaries.replace(HTML_TAG, (tag) =>
/^<\/?w:/i.test(tag) ? "" : " ",
);
- const decoded = decodeHtmlEntities(withoutTags);
- return decoded
+ return decodeHtmlEntities(withoutTags)
.split("\n")
.map((line) => {
if (!line.trim()) return "";
@@ -209,7 +380,6 @@ function pushText(segments: PostBodySegment[], raw: string, indentUnit: number):
kind: "text",
text: normalized,
...(indentLevel > 0 ? { indentLevel } : {}),
- ...(FOOTNOTE_START.test(normalized) ? { role: "footnote" as const } : {}),
});
}
}
diff --git a/lineageweave/__init__.py b/lineageweave/__init__.py
index 72e8bbcab..3122f99dd 100644
--- a/lineageweave/__init__.py
+++ b/lineageweave/__init__.py
@@ -55,4 +55,4 @@
"sentence_excerpts",
]
-__version__ = "2.12.17"
+__version__ = "2.12.18"
diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py
index d52c76297..e5789a1ea 100644
--- a/lineageweave/chunking.py
+++ b/lineageweave/chunking.py
@@ -97,11 +97,11 @@
# readable and attributable as one unit.
_TABLE_ROW_TAGS = frozenset({"tr", "w:tr"})
_TABLE_CELL_TAGS = frozenset({"td", "th", "w:tc"})
+_TABLE_TAGS = frozenset({"table", "w:tbl"})
_LIST_ITEM_START = re.compile(
r"^(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)"
)
-_FOOTNOTE_START = re.compile(r"^[*†‡](?=\S)")
def _is_footnote_block(tag: str, attrs: list[tuple[str, str | None]]) -> bool:
@@ -130,6 +130,139 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool:
)
+# Unicode Super/Subscript blocks (The Unicode Consortium, 2024, §22.4) plus the
+# Latin-1 superscript digits. Quantity display uses these so embeddings keep
+# "m³" distinct from "m3" without retaining HTML in the semantic text (ADR 0165).
+_SUPERSCRIPT = {
+ "0": "\u2070",
+ "1": "\u00b9",
+ "2": "\u00b2",
+ "3": "\u00b3",
+ "4": "\u2074",
+ "5": "\u2075",
+ "6": "\u2076",
+ "7": "\u2077",
+ "8": "\u2078",
+ "9": "\u2079",
+ "+": "\u207a",
+ "-": "\u207b",
+ "=": "\u207c",
+ "(": "\u207d",
+ ")": "\u207e",
+ "n": "\u207f",
+ "N": "\u207f",
+ "i": "\u2071",
+ "I": "\u2071",
+}
+_SUBSCRIPT = {
+ "0": "\u2080",
+ "1": "\u2081",
+ "2": "\u2082",
+ "3": "\u2083",
+ "4": "\u2084",
+ "5": "\u2085",
+ "6": "\u2086",
+ "7": "\u2087",
+ "8": "\u2088",
+ "9": "\u2089",
+ "+": "\u208a",
+ "-": "\u208b",
+ "=": "\u208c",
+ "(": "\u208d",
+ ")": "\u208e",
+ "a": "\u2090",
+ "e": "\u2091",
+ "h": "\u2095",
+ "i": "\u1d62",
+ "k": "\u2096",
+ "l": "\u2097",
+ "m": "\u2098",
+ "n": "\u2099",
+ "o": "\u2092",
+ "p": "\u209a",
+ "s": "\u209b",
+ "t": "\u209c",
+ "x": "\u2093",
+}
+_SUPERSCRIPT_VALUES = frozenset(_SUPERSCRIPT.values())
+_SUBSCRIPT_VALUES = frozenset(_SUBSCRIPT.values())
+_INLINE_SCRIPT_TAGS = frozenset({"sup", "sub"})
+_HTML_SUP = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL)
+_HTML_SUB = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL)
+_INNER_TAG = re.compile(r"<[^>]+>")
+# Quantity caret after a unit/digit, not a leading footnote marker such as `^1`.
+_CARET_EXPONENT = re.compile(
+ r"(?<=[A-Za-z0-9µμ°ΩÅåÅ)])\^(?:\{([+\-]?\d{1,3}|[nNiI])\}|([+\-]?\d{1,3}|[nNiI]))"
+)
+_ENCODED_CARET = re.compile(r"&(?:amp;)*(?:#0*94|#x0*5e);", re.IGNORECASE)
+_ENCODED_LT = r"&(?:amp;)*(?:lt|#0*60|#x0*3c);"
+_ENCODED_GT = r"&(?:amp;)*(?:gt|#0*62|#x0*3e);"
+_ENCODED_SCRIPT_TOKEN = (
+ rf"{_ENCODED_LT}\s*/?\s*(?:sup|sub)(?=\s|/|{_ENCODED_GT})"
+)
+_ENCODED_SCRIPT_PAIR = re.compile(
+ rf"{_ENCODED_LT}(?Psup|sub){_ENCODED_GT}"
+ rf"(?P(?:(?!{_ENCODED_SCRIPT_TOKEN}).)*?)"
+ rf"{_ENCODED_LT}/(?P=kind){_ENCODED_GT}",
+ re.IGNORECASE | re.DOTALL,
+)
+
+
+def apply_unicode_script(text: str, kind: str) -> str:
+ """Map a short exponent/index run to Unicode, or keep a caret/underscore."""
+ table = _SUPERSCRIPT if kind == "sup" else _SUBSCRIPT
+ values = _SUPERSCRIPT_VALUES if kind == "sup" else _SUBSCRIPT_VALUES
+ compact = text.strip()
+ if not compact:
+ return text
+ if all(ch in table or ch in values or ch.isspace() for ch in compact):
+ return "".join(table.get(ch, ch) for ch in text)
+ prefix = "^" if kind == "sup" else "_"
+ leading_len = len(text) - len(text.lstrip())
+ trailing_len = len(text) - len(text.rstrip())
+ leading = text[:leading_len]
+ trailing = text[len(text) - trailing_len :] if trailing_len else ""
+ return f"{leading}{prefix}{compact}{trailing}"
+
+
+def _decode_html_entities(text: str) -> str:
+ for _ in range(3):
+ decoded = unescape(text)
+ if decoded == text:
+ break
+ text = decoded
+ return text
+
+
+def _decode_script_entities(text: str) -> str:
+ decoded_pairs = _ENCODED_SCRIPT_PAIR.sub(
+ lambda match: (
+ f"<{match.group('kind').lower()}>{match.group('inner')}"
+ f"{match.group('kind').lower()}>"
+ ),
+ text,
+ )
+ return _ENCODED_CARET.sub(
+ lambda match: _decode_html_entities(match.group(0)), decoded_pairs
+ )
+
+
+def _replace_html_script(match: re.Match[str], kind: str) -> str:
+ inner = _decode_html_entities(match.group(1))
+ return apply_unicode_script(_INNER_TAG.sub("", inner), kind)
+
+
+def normalize_script_text(text: str) -> str:
+ """Turn HTML/caret quantity scripts into Unicode without treating comparisons as tags."""
+ replaced = _CARET_EXPONENT.sub(
+ lambda match: apply_unicode_script(match.group(1) or match.group(2), "sup"),
+ _decode_script_entities(text),
+ )
+ replaced = _HTML_SUP.sub(lambda match: _replace_html_script(match, "sup"), replaced)
+ replaced = _HTML_SUB.sub(lambda match: _replace_html_script(match, "sub"), replaced)
+ return replaced
+
+
def normalize_semantic_text(text: str) -> str:
"""Remove visual hanging-indent breaks without changing source content."""
lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
@@ -144,7 +277,7 @@ def normalize_semantic_text(text: str) -> str:
normalized[-1] = f"{normalized[-1]} {stripped}"
else:
normalized.append(stripped)
- return "\n".join(normalized).strip()
+ return normalize_script_text("\n".join(normalized).strip())
def _source_indent_width(text: str) -> int:
@@ -343,6 +476,10 @@ def __init__(self) -> None:
super().__init__()
self._stack: list[tuple[str, list[str], str | None, int, bool]] = []
self._unscoped_buffer: list[str] = []
+ self._script_stack: list[str] = []
+ self._table_cell_counts: list[int] = []
+ self._table_depth = 0
+ self._table_row_depths: list[int] = []
# Each entry is ("text", str, tag_name, style) or
# ("image", (mime_type, bytes), "", None) -- a single sequence in
# true document order, so an image's index among its siblings
@@ -351,6 +488,11 @@ def __init__(self) -> None:
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
"""Collect relevant text state when an HTML start tag is encountered."""
+ if tag in _TABLE_TAGS:
+ self._table_depth += 1
+ if tag in _INLINE_SCRIPT_TAGS:
+ self._script_stack.append(tag)
+ return
if tag == "img":
src = next((value for name, value in attrs if name == "src" and value), None)
if src:
@@ -376,16 +518,30 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None
self._stack[-1] = (tag_name, buffer, style, indent_width, True)
return
if tag in _TABLE_CELL_TAGS:
- if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS and self._stack[-1][1]:
- self._stack[-1][1].append(" | ")
+ self._script_stack.clear()
+ if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS:
+ if self._table_cell_counts[-1]:
+ self._stack[-1][1].append(" | ")
+ self._table_cell_counts[-1] += 1
return
+ if (
+ tag in _TABLE_ROW_TAGS
+ and self._table_row_depths
+ and self._table_row_depths[-1] == self._table_depth
+ ):
+ declared_width = sum(entry[3] for entry in self._stack)
+ tag_name, buffer, style, _, is_footnote = self._stack.pop()
+ self._finish_block(tag_name, buffer, style, declared_width, is_footnote)
# A rich-text editor commonly wraps a table cell in a nested
or
#
. Keep that content in the open row; otherwise the nested block
# closes first and destroys the row/column boundary.
- if any(entry[0] in _TABLE_ROW_TAGS for entry in self._stack):
+ if (
+ tag not in _TABLE_ROW_TAGS
+ and any(entry[0] in _TABLE_ROW_TAGS for entry in self._stack)
+ ):
return
if tag in _DOM_BLOCK_TAGS:
- if self._stack and self._stack[-1][1]:
+ if tag not in _TABLE_ROW_TAGS and self._stack and self._stack[-1][1]:
tag_name, buffer, style, _, is_footnote = self._stack[-1]
declared_width = sum(entry[3] for entry in self._stack)
self._finish_block(tag_name, buffer, style, declared_width, is_footnote)
@@ -397,19 +553,42 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None
self._stack.append(
(tag, [], style, _declared_indent_width(tag, attrs), is_footnote)
)
+ if tag in _TABLE_ROW_TAGS:
+ self._table_cell_counts.append(0)
+ self._table_row_depths.append(self._table_depth)
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
"""Handle self-closing block tags without losing XML indentation state."""
self.handle_starttag(tag, attrs)
- if tag in _DOM_BLOCK_TAGS:
+ if tag in _DOM_BLOCK_TAGS or tag in _INLINE_SCRIPT_TAGS or tag in _TABLE_TAGS:
self.handle_endtag(tag)
def handle_endtag(self, tag: str) -> None:
"""Close the relevant text state when an HTML end tag is encountered."""
+ if (
+ tag in _TABLE_TAGS
+ and self._table_row_depths
+ and self._table_row_depths[-1] == self._table_depth
+ ):
+ declared_width = sum(entry[3] for entry in self._stack)
+ tag_name, buffer, style, _, is_footnote = self._stack.pop()
+ self._finish_block(tag_name, buffer, style, declared_width, is_footnote)
+ if tag in _INLINE_SCRIPT_TAGS:
+ if tag in self._script_stack:
+ while self._script_stack:
+ closed = self._script_stack.pop()
+ if closed == tag:
+ break
+ return
+ if tag in _TABLE_CELL_TAGS:
+ self._script_stack.clear()
+ return
if tag in _DOM_BLOCK_TAGS and self._stack and self._stack[-1][0] == tag:
declared_width = sum(entry[3] for entry in self._stack)
tag_name, buffer, style, _, is_footnote = self._stack.pop()
self._finish_block(tag_name, buffer, style, declared_width, is_footnote)
+ if tag in _TABLE_TAGS:
+ self._table_depth = max(0, self._table_depth - 1)
def _finish_block(
self,
@@ -420,12 +599,22 @@ def _finish_block(
is_footnote: bool = False,
) -> None:
"""Emit one block buffer, including a block closed only at EOF."""
+ # An unclosed / never reaches handle_endtag, so nothing else
+ # pops it off _script_stack. Every block boundary (a sibling block
+ # opening, this block's own endtag, or EOF) routes through here, so
+ # clearing here stops a dangling script tag from bleeding into later,
+ # unrelated blocks -- mirroring how a browser would not let inline
+ # formatting survive a block-level boundary.
+ self._script_stack.clear()
raw_text = "".join(buffer)
+ if tag_name in _TABLE_ROW_TAGS:
+ self._table_cell_counts.pop()
+ self._table_row_depths.pop()
for raw_unit, source_indent in _split_dom_units(raw_text):
text = normalize_semantic_text(raw_unit)
if text:
indent_width = declared_width + source_indent
- label = "footnote" if is_footnote or _FOOTNOTE_START.match(text) else tag_name
+ label = "footnote" if is_footnote else tag_name
self._finished.append(
(
"text",
@@ -447,6 +636,8 @@ def handle_data(self, data: str) -> None:
text = decoded
had_nbsp = "\xa0" in text
text = text.replace("\xa0", " ")
+ if self._script_stack:
+ text = apply_unicode_script(text, self._script_stack[-1])
if self._stack and (text.strip() or had_nbsp):
self._stack[-1][1].append(text)
elif text.strip() or had_nbsp:
@@ -496,15 +687,31 @@ def flush() -> None:
)
+def _markdown_table_cells(line: str) -> list[str]:
+ """Return cells while removing only optional outer pipe delimiters."""
+ cells = line.strip().split("|")
+ if cells and not cells[0]:
+ cells.pop(0)
+ if cells and not cells[-1]:
+ cells.pop()
+ return cells
+
+
def _is_markdown_table_row(line: str) -> bool:
"""Recognize a pipe row only when it has at least two cells."""
- cells = line.strip().strip("|").split("|")
- return len(cells) >= 2 and all(cell.strip() for cell in cells)
+ cells = _markdown_table_cells(line)
+ return len(cells) >= 2 and any(cell.strip() for cell in cells)
+
+
+def _is_empty_markdown_table_row(line: str, column_count: int) -> bool:
+ """Recognize an all-empty row only inside an established table."""
+ cells = _markdown_table_cells(line)
+ return len(cells) == column_count and not any(cell.strip() for cell in cells)
def _render_markdown_table_row(line: str) -> str:
"""Keep Markdown table columns as searchable row evidence."""
- return " | ".join(cell.strip() for cell in line.strip().strip("|").split("|"))
+ return " | ".join(cell.strip() for cell in _markdown_table_cells(line))
def _split_plain_text_units(text: str) -> list[tuple[str, int, str]]:
@@ -531,15 +738,27 @@ def flush() -> None:
continue
if _is_markdown_table_row(line):
rows: list[str] = []
- while index < len(lines) and _is_markdown_table_row(lines[index]):
- rows.append(lines[index])
+ column_count = len(_markdown_table_cells(line))
+ while index < len(lines):
+ candidate = lines[index]
+ established = (
+ len(rows) >= 2
+ and bool(_MARKDOWN_TABLE_SEPARATOR.match(rows[1]))
+ and len(_markdown_table_cells(rows[1])) == column_count
+ )
+ if not _is_markdown_table_row(candidate) and not (
+ established
+ and _is_empty_markdown_table_row(candidate, column_count)
+ ):
+ break
+ rows.append(candidate)
index += 1
data_rows = [row for row in rows if not _MARKDOWN_TABLE_SEPARATOR.match(row)]
if len(data_rows) >= 2:
flush()
units.extend(
(
- _render_markdown_table_row(row),
+ normalize_semantic_text(_render_markdown_table_row(row)),
_source_indent_width(row),
"tr",
)
diff --git a/lineageweave/post_content_normalization.py b/lineageweave/post_content_normalization.py
index 196b7e7b4..000d06c40 100644
--- a/lineageweave/post_content_normalization.py
+++ b/lineageweave/post_content_normalization.py
@@ -42,6 +42,7 @@
_HTML_OPEN_TAG = re.compile(
r"<\s*/?\s*(?:article|section|nav|aside|header|footer|div|p|li|td|th|tr|"
r"table|blockquote|h[1-6]|img|br|hr|ul|ol|span|strong|em|b|i|u|a|"
+ r"sup|sub|"
r"html|body|head|style|script|font|center|pre)\b",
re.IGNORECASE,
)
diff --git a/pyproject.toml b/pyproject.toml
index 06e27ac93..9a3975230 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "lineageweave"
-version = "2.12.17"
+version = "2.12.18"
description = "Reconstructs git-branch-style lineage DAGs from scattered short records using multi-channel score fusion and LLM adjudication."
readme = "README.md"
license = { text = "MIT" }
diff --git a/tests/test_chunking.py b/tests/test_chunking.py
index d37a300cc..b7be87013 100644
--- a/tests/test_chunking.py
+++ b/tests/test_chunking.py
@@ -7,6 +7,7 @@
chunk_by_paragraph,
chunk_by_sentence,
chunk_by_source_body,
+ normalize_script_text,
normalize_semantic_text,
)
@@ -104,11 +105,88 @@ def test_chunk_by_dom_keeps_nested_table_cell_blocks_in_their_row() -> None:
assert [(chunk.label, chunk.text) for chunk in chunks] == [("tr", "No. | Company")]
-def test_chunk_by_dom_labels_markerless_footnotes() -> None:
- chunks = chunk_by_dom("
1. 배경 '
@@ -410,3 +552,63 @@ def test_chunk_by_dom_preserves_style_per_block_independently() -> None:
assert chunks[0].style == "color:blue"
assert chunks[1].style is None
+
+
+def test_normalize_script_text_maps_quantity_exponents_and_leaves_comparisons() -> None:
+ assert normalize_script_text("Tank volume is 12 m3.") == "Tank volume is 12 m³."
+ assert normalize_script_text("Tank volume is 12 m^3.") == "Tank volume is 12 m³."
+ assert normalize_script_text("Coolant is H2O.") == "Coolant is H₂O."
+ assert normalize_script_text("x") == "x "
+ assert normalize_script_text("qty < 50 and price > 10") == "qty < 50 and price > 10"
+ assert normalize_script_text("^1 See the tank note.") == "^1 See the tank note."
+
+
+def test_normalize_script_text_keeps_mixed_script_content_as_a_visible_fallback() -> None:
+ assert normalize_script_text("x3a") == "x^3a"
+
+
+def test_normalize_script_text_decodes_nested_inline_markup_before_stripping() -> None:
+ assert normalize_script_text("x<span>2</span>") == "x²"
+
+
+def test_chunk_by_dom_keeps_html_quantity_scripts_as_unicode() -> None:
+ chunks = chunk_by_dom("
Tank volume is 12 m3 of H2O.
")
+
+ assert [chunk.text for chunk in chunks] == ["Tank volume is 12 m³ of H₂O."]
+
+
+def test_chunk_by_dom_normalizes_entity_encoded_quantity_scripts() -> None:
+ chunks = chunk_by_dom(
+ "
Reserve 12 m^3 and x<sup>2</sup> units.
"
+ )
+
+ assert [chunk.text for chunk in chunks] == ["Reserve 12 m³ and x² units."]
+
+
+def test_chunk_by_dom_unclosed_sup_does_not_cross_table_cells() -> None:
+ chunks = chunk_by_dom("
m3
Acme Corp
")
+
+ assert [chunk.text for chunk in chunks] == ["m³ | Acme Corp"]
+
+
+def test_chunk_by_dom_unclosed_sup_does_not_corrupt_later_paragraphs() -> None:
+ """A malformed, never-closed must not leak its script context into
+ every later block. HTMLParser (unlike a browser) does not implicitly
+ close an unclosed inline tag at a block boundary, so a naive
+ _script_stack would otherwise stay "open" for the rest of the document."""
+ html = (
+ "
Tank volume is 12 m3
"
+ "
Unrelated paragraph mentions n2 and o2 plainly.
"
+ )
+ chunks = chunk_by_dom(html)
+
+ assert [chunk.text for chunk in chunks] == [
+ "Tank volume is 12 m³",
+ "Unrelated paragraph mentions n2 and o2 plainly.",
+ ]
+
+
+def test_chunk_by_source_body_maps_plain_caret_quantities() -> None:
+ chunks = chunk_by_source_body("Reserve 12 m^3 and 10^{-3} M stock.")
+
+ assert chunks[0].text == "Reserve 12 m³ and 10⁻³ M stock."
diff --git a/tests/test_post_content_normalization.py b/tests/test_post_content_normalization.py
index 0beead6f4..3c7c7c0d7 100644
--- a/tests/test_post_content_normalization.py
+++ b/tests/test_post_content_normalization.py
@@ -375,6 +375,13 @@ def test_comparison_operators_in_plain_text_are_not_treated_as_html() -> None:
assert result.formatting_hints == ()
+def test_quantity_superscripts_normalize_to_unicode_for_embeddings() -> None:
+ html = normalize_post_body("