Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions apps/server/src/provider/documentTextWorker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ type TextPart = string | typeof CELL_START | typeof CELL_END | { fallback: strin

const TRUNCATED = "\n\n[Extracted text truncated at document extraction limits.]";
// Parts the document never displays: the XHTML head, tracked deletions and moves, field codes,
// phonetic guides, paragraph properties such as tab-stop definitions.
// phonetic guides, paragraph properties such as tab-stop definitions, drawing anchor layout
// values and encoded field data.
const HIDDEN = new Set([
"head",
"script",
Expand All @@ -28,6 +29,11 @@ const HIDDEN = new Set([
"delInstrText",
"tracked-changes",
"pPr",
"positionH",
"positionV",
"sizeRelH",
"sizeRelV",
"fldData",
]);

// @effect-diagnostics-next-line extendsNativeError:off - this worker loads no Effect runtime; Truncation is a local throw sentinel.
Expand Down Expand Up @@ -109,7 +115,7 @@ function xml(source: string, extension: string): XmlNode {
parent.children.pop();
return;
}
if (!node.hasText && node.name !== "noBreakHyphen") return;
if (HIDDEN.has(node.name) || (!node.hasText && node.name !== "noBreakHyphen")) return;
parent.hasText = true;
if (node.name === "Choice") parent.hasTextChoice = true;
});
Expand Down
31 changes: 31 additions & 0 deletions apps/server/src/provider/userInputAttachments.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,37 @@ describe("question answer paths", () => {
Effect.gen(function* () {
const config = yield* ServerConfig.ServerConfig;
const cases = [
// Word stores object layout and field data outside the visible document text.
{
ext: "docx",
bounded: true,
parts: {
"word/document.xml":
'<w:document xmlns:w="urn:w" xmlns:wp="urn:wp" xmlns:wp14="urn:wp14" xmlns:mc="urn:mc"><w:body><w:p><w:r><w:t>Total cost is </w:t><w:drawing><wp:anchor><wp:positionH><wp:posOffset>1905000</wp:posOffset></wp:positionH><wp:positionV><wp:posOffset>-635</wp:posOffset></wp:positionV><wp14:sizeRelH><wp14:pctWidth>0</wp14:pctWidth></wp14:sizeRelH><wp14:sizeRelV><wp14:pctHeight>0</wp14:pctHeight></wp14:sizeRelV></wp:anchor></w:drawing><w:t>42 EUR.</w:t></w:r></w:p></w:body></w:document>',
},
expected: ["Total cost is 42 EUR."],
absent: ["1905000", "-635", "0042"],
},
{
ext: "docx",
bounded: true,
parts: {
"word/document.xml":
'<w:document xmlns:w="urn:w" xmlns:wp="urn:wp" xmlns:wp14="urn:wp14" xmlns:mc="urn:mc"><w:body><w:p><mc:AlternateContent><mc:Choice Requires="cx"><w:drawing><wp:anchor><wp:positionH><wp:posOffset>1905000</wp:posOffset></wp:positionH><wp:positionV><wp:posOffset>-635</wp:posOffset></wp:positionV><wp14:sizeRelH><wp14:pctWidth>0</wp14:pctWidth></wp14:sizeRelH><wp14:sizeRelV><wp14:pctHeight>0</wp14:pctHeight></wp14:sizeRelV></wp:anchor></w:drawing><cx:chart/></mc:Choice><mc:Fallback><w:pict><v:shape><v:textbox><w:txbxContent><w:p><w:r><w:t>Chart caption</w:t></w:r></w:p></w:txbxContent></v:textbox></v:shape></w:pict></mc:Fallback></mc:AlternateContent></w:p></w:body></w:document>',
},
expected: ["Chart caption"],
absent: ["1905000", "-635"],
},
{
ext: "docx",
bounded: true,
parts: {
"word/document.xml":
'<w:document xmlns:w="urn:w" xmlns:wp="urn:wp" xmlns:wp14="urn:wp14" xmlns:mc="urn:mc"><w:body><w:p><w:r><w:t>As shown </w:t><w:fldChar><w:fldData>RU5ETk9URQ==</w:fldData></w:fldChar><w:t>(Smith 2020) earlier.</w:t></w:r></w:p></w:body></w:document>',
},
expected: ["As shown (Smith 2020) earlier."],
absent: ["RU5ETk9URQ=="],
},
{
ext: "docx",
bounded: true,
Expand Down
Loading