-
Notifications
You must be signed in to change notification settings - Fork 1
fix(frontend): preserve structured footnote roles #388
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
f145b83
81cd12f
146cc56
ee5882a
1b680a2
caea21b
86ac1d4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 📝 Info: Marker hides footnote-line indentation from unit inference
(Refers to this code) Was this helpful? React with 👍 or 👎 to provide feedback. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -24,10 +24,62 @@ const FOOTNOTE_START = /^\s*[*†‡](?=\S)/; | |
| const INDENT_MARKER = "\u0001lw-indent:"; | ||
| const INDENT_MARKER_END = "\u0002"; | ||
| const INDENT_MARKER_PATTERN = /lw-indent:(\d+)/g; | ||
| const FOOTNOTE_MARKER = "\u0001lw-footnote\u0002"; | ||
| const FOOTNOTE_MARKER_PATTERN = new RegExp(FOOTNOTE_MARKER, "g"); | ||
|
|
||
| function markFootnoteTags(markup: string): string { | ||
| let footnoteDepth = 0; | ||
| const openTags: Array<{ name: string; isFootnote: boolean }> = []; | ||
| const voidTags = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "w:br"]); | ||
| return markup.replace(HTML_TAG, (tag) => { | ||
| const match = tag.match(/^<\s*(\/?)\s*([a-z][a-z0-9:-]*)\b/i); | ||
| if (!match) return tag; | ||
| const closing = Boolean(match[1]); | ||
| const name = match[2].toLowerCase(); | ||
| const hasFootnoteLabel = [...tag.matchAll(/\b(?:class|role)\s*=\s*(["'])(.*?)\1/gi)].some( | ||
| (attribute) => | ||
| /\b(?:footnotes?|endnotes?|msofootnotetext|msoendnotetext)\b/i.test(attribute[2]), | ||
| ); | ||
| const isContainer = | ||
| hasFootnoteLabel && (name === "div" || name === "ol" || name === "ul"); | ||
| const isWordParagraph = | ||
| name === "p" && hasFootnoteLabel; | ||
| const isOoxmlContainer = name === "w:footnote" || name === "w:endnote"; | ||
|
devin-ai-integration[bot] marked this conversation as resolved.
|
||
|
|
||
| if (closing) { | ||
| const matchingIndex = openTags.map((entry) => entry.name).lastIndexOf(name); | ||
| if (matchingIndex >= 0) { | ||
| const closedTags = openTags.splice(matchingIndex); | ||
| footnoteDepth = Math.max( | ||
| 0, | ||
| footnoteDepth - closedTags.filter((entry) => entry.isFootnote).length, | ||
| ); | ||
| } | ||
| return tag; | ||
| } | ||
|
devin-ai-integration[bot] marked this conversation as resolved.
|
||
| const selfClosing = /\/\s*>$/.test(tag) || voidTags.has(name); | ||
| const opensFootnote = isOoxmlContainer || isContainer; | ||
| if (!selfClosing) { | ||
| openTags.push({ name, isFootnote: opensFootnote }); | ||
| } | ||
| if (opensFootnote) { | ||
| if (!selfClosing) footnoteDepth += 1; | ||
| return `${tag}${FOOTNOTE_MARKER}`; | ||
| } | ||
| if ( | ||
| isWordParagraph || | ||
| (footnoteDepth > 0 && (name === "li" || name === "p" || name === "w:p")) | ||
| ) { | ||
| return `${tag}${FOOTNOTE_MARKER}`; | ||
| } | ||
|
Comment on lines
+61
to
+74
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔍 Unclosed footnote container mislabels all later paragraphs
Was this helpful? React with 👍 or 👎 to provide feedback. |
||
| return tag; | ||
| }); | ||
| } | ||
|
|
||
| function stripIndentMarkers(value: string): string { | ||
| return value | ||
| .replace(INDENT_MARKER_PATTERN, "") | ||
| .replace(FOOTNOTE_MARKER_PATTERN, "") | ||
| .split(String.fromCharCode(1)) | ||
| .join("") | ||
| .split(String.fromCharCode(2)) | ||
|
|
@@ -90,17 +142,18 @@ function indentMarker(width: number): string { | |
| } | ||
|
|
||
| function stripHtmlTags(text: string): string { | ||
| text = text.replace(/<sup[^>]*>(.*?)<\/sup>/gi, "^$1"); | ||
| text = markFootnoteTags(text).replace(/<sup[^>]*>(.*?)<\/sup>/gi, "^$1"); | ||
| const withBoundaries = text | ||
| .replace(BREAK_TAG, "\n") | ||
| .replace(BLOCK_TAG, (tag) => { | ||
| if (/^<\//.test(tag)) return "\n\n"; | ||
| return `\n\n${indentMarker(declaredIndentWidth(tag))}`; | ||
| }) | ||
| .replace(WORD_INDENT_TAG, (tag) => indentMarker(declaredIndentWidth(tag))); | ||
| const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => | ||
| /^<\/?w:/i.test(tag) ? "" : " ", | ||
| ); | ||
| const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => { | ||
| if (/^<\/?(?:a\b|w:)/i.test(tag)) return ""; | ||
| return " "; | ||
| }); | ||
|
seonghobae marked this conversation as resolved.
seonghobae marked this conversation as resolved.
|
||
| const decoded = decodeHtmlEntities(withoutTags); | ||
| return decoded | ||
| .split("\n") | ||
|
|
@@ -200,6 +253,7 @@ function isDecodableBase64(raw: string): boolean { | |
| function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void { | ||
| const text = stripHtmlTags(raw); | ||
| for (const paragraph of splitSemanticParagraphs(text)) { | ||
| const isMarkedFootnote = paragraph.includes(FOOTNOTE_MARKER); | ||
| const indentLevel = indentationLevel(paragraph, indentUnit); | ||
| const normalized = stripIndentMarkers(paragraph) | ||
| .replace(/^[ \t]+/, "") | ||
|
|
@@ -209,7 +263,9 @@ function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): | |
| kind: "text", | ||
| text: normalized, | ||
| ...(indentLevel > 0 ? { indentLevel } : {}), | ||
| ...(FOOTNOTE_START.test(normalized) ? { role: "footnote" as const } : {}), | ||
| ...(isMarkedFootnote || FOOTNOTE_START.test(normalized) | ||
| ? { role: "footnote" as const } | ||
| : {}), | ||
| }); | ||
| } | ||
| } | ||
|
|
@@ -243,7 +299,7 @@ export function splitPostBody(body: string): PostBodySegment[] { | |
| } | ||
| pushText(segments, body.slice(lastIndex), indentUnit); | ||
| if (segments.length === 0) { | ||
| return [{ kind: "text", text: stripHtmlTags(body) }]; | ||
| return [{ kind: "text", text: stripIndentMarkers(stripHtmlTags(body)) }]; | ||
|
seonghobae marked this conversation as resolved.
|
||
| } | ||
| return segments; | ||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.