Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions frontend/src/postBodyDisplay.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,69 @@ describe("splitPostBody", () => {
]);
});

it("labels HTML, Word, and OOXML footnotes in the fallback renderer", () => {
expect(
splitPostBody(
'<p>Body text</p>' +
'<ol class="footnotes"><li id="fn1"><p>HTML footnote body</p></li></ol>' +
'<p class="MsoFootnoteText"><a href="#_ftnref1"><sup>1</sup></a> Word footnote body</p>' +
"<w:footnote w:id='1'><w:p>OOXML footnote body</w:p></w:footnote>",
),
).toEqual([
{ kind: "text", text: "Body text" },
{ kind: "text", text: "HTML footnote body", role: "footnote" },
{ kind: "text", text: "^1 Word footnote body", role: "footnote" },
{ kind: "text", text: "OOXML footnote body", role: "footnote" },
]);
});

it("stops labeling ordinary content after an HTML footnote list", () => {
expect(
splitPostBody(
'<ol class="footnotes"><li>HTML footnote body</li></ol><p>Ordinary body after footnotes</p>',
),
).toEqual([
{ kind: "text", text: "HTML footnote body", role: "footnote" },
{ kind: "text", text: "Ordinary body after footnotes" },
]);
});

it("labels footnotes inside a labeled wrapper around an HTML list", () => {
expect(
splitPostBody(
'<p>Body text</p>' +
'<div class="footnotes"><ol><li><p>Wrapped footnote body</p></li></ol></div>' +
"<p>Ordinary body after footnotes</p>",
),
).toEqual([
{ kind: "text", text: "Body text" },
{ kind: "text", text: "Wrapped footnote body", role: "footnote" },
{ kind: "text", text: "Ordinary body after footnotes" },
]);
});

it("does not expose control markers for an empty footnote container", () => {
expect(splitPostBody('<ol class="footnotes"></ol>')).toEqual([{ kind: "text", text: "" }]);
});

it("does not infer footnotes from unrelated attribute values", () => {
expect(
splitPostBody(
'<ol data-purpose="footnotes"><li>Ordinary list</li></ol>' +
'<p data-purpose="footnote">Ordinary paragraph</p>',
),
).toEqual([
{ kind: "text", text: "Ordinary list" },
{ kind: "text", text: "Ordinary paragraph" },
]);
});

it("keeps text boundaries for tags whose names start with a", () => {
expect(splitPostBody('<p>Alpha<abbr title="expanded">Beta</abbr>Gamma</p>')).toEqual([
{ kind: "text", text: "Alpha Beta Gamma" },
]);
});

it("leaves a plain-text post unchanged so existing popups keep their wording", () => {
expect(splitPostBody("The full body text.")).toEqual([
{ kind: "text", text: "The full body text." },
Expand Down
68 changes: 62 additions & 6 deletions frontend/src/postBodyDisplay.ts
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 Info: Marker hides footnote-line indentation from unit inference

inferIndentationUnit measures leading whitespace without stripping the footnote marker, so a marker-prefixed footnote line reports width 0. indentationLevel strips markers first, then measures. The two paths disagree on the same line's indentation. Impact is small since footnotes are rarely indented.

(Refers to this code)

Open in Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,62 @@ const FOOTNOTE_START = /^\s*[*†‡](?=\S)/;
const INDENT_MARKER = "\u0001lw-indent:";
const INDENT_MARKER_END = "\u0002";
const INDENT_MARKER_PATTERN = /lw-indent:(\d+)/g;
const FOOTNOTE_MARKER = "\u0001lw-footnote\u0002";
const FOOTNOTE_MARKER_PATTERN = new RegExp(FOOTNOTE_MARKER, "g");

function markFootnoteTags(markup: string): string {
let footnoteDepth = 0;
const openTags: Array<{ name: string; isFootnote: boolean }> = [];
const voidTags = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "w:br"]);
return markup.replace(HTML_TAG, (tag) => {
const match = tag.match(/^<\s*(\/?)\s*([a-z][a-z0-9:-]*)\b/i);
if (!match) return tag;
const closing = Boolean(match[1]);
const name = match[2].toLowerCase();
const hasFootnoteLabel = [...tag.matchAll(/\b(?:class|role)\s*=\s*(["'])(.*?)\1/gi)].some(
(attribute) =>
/\b(?:footnotes?|endnotes?|msofootnotetext|msoendnotetext)\b/i.test(attribute[2]),
);
const isContainer =
hasFootnoteLabel && (name === "div" || name === "ol" || name === "ul");
const isWordParagraph =
name === "p" && hasFootnoteLabel;
const isOoxmlContainer = name === "w:footnote" || name === "w:endnote";
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.

if (closing) {
const matchingIndex = openTags.map((entry) => entry.name).lastIndexOf(name);
if (matchingIndex >= 0) {
const closedTags = openTags.splice(matchingIndex);
footnoteDepth = Math.max(
0,
footnoteDepth - closedTags.filter((entry) => entry.isFootnote).length,
);
}
return tag;
}
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.
const selfClosing = /\/\s*>$/.test(tag) || voidTags.has(name);
const opensFootnote = isOoxmlContainer || isContainer;
if (!selfClosing) {
openTags.push({ name, isFootnote: opensFootnote });
}
if (opensFootnote) {
if (!selfClosing) footnoteDepth += 1;
return `${tag}${FOOTNOTE_MARKER}`;
}
if (
isWordParagraph ||
(footnoteDepth > 0 && (name === "li" || name === "p" || name === "w:p"))
) {
return `${tag}${FOOTNOTE_MARKER}`;
}
Comment on lines +61 to +74

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔍 Unclosed footnote container mislabels all later paragraphs

markFootnoteTags raises footnoteDepth on an opening footnote container and only lowers it on a matching close. If a container is never closed in malformed markup, the depth stays positive and every later <li>/<p>/<w:p> is tagged role: "footnote". Balanced generator/Word/OOXML output avoids this; hand-authored or truncated HTML does not.

Open in Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

return tag;
});
}

function stripIndentMarkers(value: string): string {
return value
.replace(INDENT_MARKER_PATTERN, "")
.replace(FOOTNOTE_MARKER_PATTERN, "")
.split(String.fromCharCode(1))
.join("")
.split(String.fromCharCode(2))
Expand Down Expand Up @@ -90,17 +142,18 @@ function indentMarker(width: number): string {
}

function stripHtmlTags(text: string): string {
text = text.replace(/<sup[^>]*>(.*?)<\/sup>/gi, "^$1");
text = markFootnoteTags(text).replace(/<sup[^>]*>(.*?)<\/sup>/gi, "^$1");
const withBoundaries = text
.replace(BREAK_TAG, "\n")
.replace(BLOCK_TAG, (tag) => {
if (/^<\//.test(tag)) return "\n\n";
return `\n\n${indentMarker(declaredIndentWidth(tag))}`;
})
.replace(WORD_INDENT_TAG, (tag) => indentMarker(declaredIndentWidth(tag)));
const withoutTags = withBoundaries.replace(HTML_TAG, (tag) =>
/^<\/?w:/i.test(tag) ? "" : " ",
);
const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => {
if (/^<\/?(?:a\b|w:)/i.test(tag)) return "";
return " ";
});
Comment thread
seonghobae marked this conversation as resolved.
Comment thread
seonghobae marked this conversation as resolved.
const decoded = decodeHtmlEntities(withoutTags);
return decoded
.split("\n")
Expand Down Expand Up @@ -200,6 +253,7 @@ function isDecodableBase64(raw: string): boolean {
function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void {
const text = stripHtmlTags(raw);
for (const paragraph of splitSemanticParagraphs(text)) {
const isMarkedFootnote = paragraph.includes(FOOTNOTE_MARKER);
const indentLevel = indentationLevel(paragraph, indentUnit);
const normalized = stripIndentMarkers(paragraph)
.replace(/^[ \t]+/, "")
Expand All @@ -209,7 +263,9 @@ function pushText(segments: PostBodySegment[], raw: string, indentUnit: number):
kind: "text",
text: normalized,
...(indentLevel > 0 ? { indentLevel } : {}),
...(FOOTNOTE_START.test(normalized) ? { role: "footnote" as const } : {}),
...(isMarkedFootnote || FOOTNOTE_START.test(normalized)
? { role: "footnote" as const }
: {}),
});
}
}
Expand Down Expand Up @@ -243,7 +299,7 @@ export function splitPostBody(body: string): PostBodySegment[] {
}
pushText(segments, body.slice(lastIndex), indentUnit);
if (segments.length === 0) {
return [{ kind: "text", text: stripHtmlTags(body) }];
return [{ kind: "text", text: stripIndentMarkers(stripHtmlTags(body)) }];
Comment thread
seonghobae marked this conversation as resolved.
}
return segments;
}