diff --git a/packages/pdf/src/tables.ts b/packages/pdf/src/tables.ts index 3c21edc..93b5861 100644 --- a/packages/pdf/src/tables.ts +++ b/packages/pdf/src/tables.ts @@ -37,16 +37,19 @@ export interface DetectedTable { const SNAP = 2.5; const THIN = 3; -const MIN_RULE = 24; +const MIN_RULE = 12; const COL_GAP_FACTOR = 1.0; const MIN_COLUMNS = 2; const MIN_ROWS = 2; -const MAX_TABLE_ROWS = 40; +const MAX_TABLE_ROWS = 60; const MAX_CELL_WORDS = 6; const MAX_KEY_VALUE_WORDS = 40; -const MAX_TABLE_WORDS = 400; +const MAX_TABLE_WORDS = 1500; const PROSE_AVG_CELL_CHARS = 18; const PAGE_COVER = 0.7; +const OUTER_EDGE = 8; +const RUN_GAP = 12 * 3.2; +const WRAP_SPAN = 0.72; export function detectTables( items: TableTextItem[], @@ -196,22 +199,31 @@ function buildGridTable( horiz.map((h) => h.y), SNAP, ).sort((a, b) => b - a); - if (xs.length < 3 || ys.length < 3) return undefined; + if (xs.length < 2 || ys.length < 2) return undefined; + + const hLeft = Math.min(...horiz.map((h) => h.x1)); + const hRight = Math.max(...horiz.map((h) => h.x2)); + const colEdges = xs.slice(); + if (colEdges[0]! - hLeft > OUTER_EDGE) colEdges.unshift(hLeft); + if (hRight - colEdges[colEdges.length - 1]! > OUTER_EDGE) colEdges.push(hRight); - const colEdges = xs; - const rowEdges = ys; const nCols = colEdges.length - 1; - const nRows = rowEdges.length - 1; - if (nCols < 2 || nRows < 2 || nCols > 20 || nRows > 60) return undefined; + const nRows = ys.length - 1; + if (nCols < 2 || nCols > 20 || nRows > 60) return undefined; + + const xLeft = colEdges[0]!; + const xRight = colEdges[colEdges.length - 1]!; + const yTop = Math.max(ys[0]!, ...vert.map((v) => v.y2)); + const yBottom = Math.min(ys[ys.length - 1]!, ...vert.map((v) => v.y1)); + + if (nRows < 2) { + return buildTextGrid(items, colEdges, xLeft, xRight, yTop, yBottom, page, claimed); + } const cellItems: { idx: number; item: TableTextItem }[][][] = Array.from({ length: nRows }, () => Array.from({ length: nCols }, () => []), ); const used: number[] = []; - const xLeft = colEdges[0]!; - const xRight = colEdges[colEdges.length - 1]!; - const yTop = rowEdges[0]!; - const yBottom = rowEdges[rowEdges.length - 1]!; for (let idx = 0; idx < items.length; idx += 1) { if (claimed.has(idx)) continue; @@ -221,7 +233,7 @@ function buildGridTable( const cy = item.y; if (cx < xLeft - 4 || cx > xRight + 4 || cy < yBottom - 4 || cy > yTop + 4) continue; const col = findBand(colEdges, cx, false); - const row = findBand(rowEdges, cy, true); + const row = findBand(ys, cy, true); if (col === undefined || row === undefined) continue; cellItems[row]![col]!.push({ idx, item }); used.push(idx); @@ -255,17 +267,53 @@ function buildGridTable( }; } +function buildTextGrid( + items: TableTextItem[], + colEdges: number[], + xLeft: number, + xRight: number, + yTop: number, + yBottom: number, + page: number, + claimed: Set, +): DetectedTable | undefined { + const outside = new Set(claimed); + for (let idx = 0; idx < items.length; idx += 1) { + if (outside.has(idx)) continue; + const item = items[idx]!; + if (item.page !== page || item.text.trim().length === 0) { + outside.add(idx); + continue; + } + const cx = item.x + Math.max(item.width, 0) / 2; + const cy = item.y; + if (cx < xLeft - 4 || cx > xRight + 4 || cy < yBottom - 4 || cy > yTop + 4) outside.add(idx); + } + const rows = collectAlignedRows(items, page, outside); + if (rows.length < MIN_ROWS) return undefined; + const colXs = colEdges.slice(0, -1).map((x, i) => (x + colEdges[i + 1]!) / 2); + return finishAlignedTable(rows, colXs, page, undefined, colEdges); +} + function acceptGrid(grid: string[][]): boolean { + const nCols = grid[0]!.length; const filled = grid.flat().filter((c) => c.length > 0); - const fillRate = filled.length / (grid.length * grid[0]!.length); + const fillRate = filled.length / (grid.length * nCols); if (fillRate < 0.15) return false; const multi = grid.filter((row) => row.filter((c) => c.length > 0).length >= 2).length; if (multi * 2 < grid.length) return false; const lengths = filled.map((c) => [...c].length); const avg = lengths.reduce((a, b) => a + b, 0) / lengths.length; - if (avg > 80) return false; + const colAvgs = Array.from({ length: nCols }, (_, c) => { + const cells = grid.map((row) => row[c]!).filter((cell) => cell.length > 0); + if (cells.length === 0) return 0; + return cells.reduce((n, cell) => n + [...cell].length, 0) / cells.length; + }); + const longCols = colAvgs.filter((n) => n > 80).length; + if (avg > 80 && longCols >= nCols) return false; + if (avg > 220) return false; const long = lengths.filter((n) => n > 160).length; - if (long > 0 && long / lengths.length > 0.2) return false; + if (long > 0 && long / lengths.length > 0.2 && longCols > 2) return false; return true; } @@ -307,16 +355,16 @@ function detectBorderlessTables( const prev = run[run.length - 1]!; const cur = rows[j]!; const gap = prev.y - cur.y; - const fs = 12; - if (gap > fs * 3.2) break; - if (!isTableRow(cur)) break; - run.push(cur); - j += 1; - } - if (run.length >= MIN_ROWS) { - const table = buildAlignedTable(run, page, pageBox); - if (table) out.push(table); + if (gap > RUN_GAP) break; + if (isTableRow(cur) || isContinuation(run, cur)) { + run.push(cur); + j += 1; + continue; + } + break; } + const table = buildAlignedTable(run, page, pageBox); + if (table) out.push(table); i = Math.max(j, i + 1); } return out; @@ -414,39 +462,150 @@ function isTableRow(row: RowCand): boolean { return true; } -function buildAlignedTable( - run: RowCand[], - page: number, - pageBox: { x: number; y: number; x2: number; y2: number } | undefined, -): DetectedTable | undefined { - if (run.length < MIN_ROWS || run.length > MAX_TABLE_ROWS) return undefined; - const counts = run.map((r) => r.segs.length); +function isCaption(row: RowCand): boolean { + const text = row.segs + .map((s) => s.text) + .join(' ') + .trim(); + return /^(source|note|notes|table|fig\.?|figure)\b[:\s]/i.test(text); +} + +function isContinuation(run: RowCand[], cur: RowCand): boolean { + if (isCaption(cur)) return false; + const prev = run[run.length - 1]!; + if (prev.y - cur.y > RUN_GAP * 0.7) return false; + if (cur.segs.some((s) => s.words > MAX_KEY_VALUE_WORDS)) return false; + const full = run.filter((r) => r.segs.length >= MIN_COLUMNS); + if (full.length === 0) return false; + const template = full.reduce((a, b) => (b.segs.length >= a.segs.length ? b : a)); + const colXs = template.segs.map((s) => (s.x + s.x2) / 2); + const tableX1 = Math.min(...run.flatMap((r) => r.segs.map((s) => s.x))); + const tableX2 = Math.max(...run.flatMap((r) => r.segs.map((s) => s.x2))); + const tableW = Math.max(tableX2 - tableX1, 1); + if (cur.segs.length === 1) { + const seg = cur.segs[0]!; + if ((seg.x2 - seg.x) / tableW > WRAP_SPAN) return false; + } + return cur.segs.every((s) => { + const mid = (s.x + s.x2) / 2; + const col = nearestCol(colXs, mid); + return Math.abs(mid - colXs[col]!) < tableW * 0.4; + }); +} + +function inferColumns(run: RowCand[]): { colXs: number[]; edges: number[] } | undefined { + const full = run.filter((r) => r.segs.length >= MIN_COLUMNS); + if (full.length === 0) return undefined; + const counts = full.map((r) => r.segs.length); const maxCols = Math.max(...counts); const nCols = maxCols >= 3 ? maxCols : medianInt(counts); if (nCols < MIN_COLUMNS) return undefined; const matching = counts.filter((c) => c === nCols).length; if (nCols >= 3) { if (matching < 1) return undefined; - } else if (matching / run.length < 0.5) { + } else if (matching / full.length < 0.5) { return undefined; } - - const header = run.find((r) => r.segs.length === nCols) ?? run[0]!; + const header = full.find((r) => r.segs.length === nCols) ?? full[0]!; const colXs = header.segs.map((s) => (s.x + s.x2) / 2); + const edges = [...header.segs.map((s) => s.x), header.segs[header.segs.length - 1]!.x2]; + return { colXs, edges }; +} + +function edgesFromCenters(run: RowCand[], colXs: number[]): number[] | undefined { + const header = run.find((r) => r.segs.length === colXs.length); + if (header) { + return [...header.segs.map((s) => s.x), header.segs[header.segs.length - 1]!.x2]; + } + return undefined; +} + +function appendCell(row: string[], col: number, text: string): void { + const prev = row[col]!; + row[col] = prev.length === 0 ? text : `${prev} ${text}`; +} +function columnIndex(x: number, colXs: number[], edges?: number[]): number { + if (edges && edges.length === colXs.length + 1) { + const band = findBand(edges, x, false); + if (band !== undefined) return band; + } + for (let i = 0; i < colXs.length - 1; i += 1) { + if (x < (colXs[i]! + colXs[i + 1]!) / 2) return i; + } + return colXs.length - 1; +} + +function foldRows( + run: RowCand[], + colXs: number[], + edges?: number[], +): { grid: string[][]; used: number[] } { + const nCols = colXs.length; const grid: string[][] = []; const used: number[] = []; - for (const row of run) { - const cells = Array.from({ length: nCols }, () => ''); - for (const seg of row.segs) { - const col = nearestCol(colXs, (seg.x + seg.x2) / 2); - const prev = cells[col]!; - cells[col] = prev.length === 0 ? seg.text : `${prev} ${seg.text}`; + let current: string[] | undefined; + + const colOf = (seg: CellSeg): number => columnIndex((seg.x + seg.x2) / 2, colXs, edges); + const filledCount = (row: string[]): number => row.filter((c) => c.length > 0).length; + const mostlyFilled = (row: string[]): boolean => { + const n = filledCount(row); + return n >= nCols - 1 || (n >= 2 && row[0]!.length > 0); + }; + + const startsRow = (line: RowCand): boolean => { + if (!current) return true; + const cols = [...new Set(line.segs.map(colOf))]; + const hasKey = cols.includes(0); + if (!hasKey) { + const text = line.segs + .map((s) => s.text) + .join(' ') + .trim(); + if (/^(total|sum|subtotal|average|overall)\b/i.test(text) && mostlyFilled(current)) { + return true; + } + if (!mostlyFilled(current)) return false; + if (grid.length === 0) return false; + if (cols.some((c) => wordCount(current![c]!) >= 3)) return false; + if (line.segs.every((s) => /^[a-z(]/.test(s.text.trim()))) return false; + if (cols.length === 1 && current![cols[0]!]!.length > 0) return false; + return true; + } + if (current[0]!.length === 0) return false; + if (cols.length === 1) return false; + const key = line.segs.find((s) => colOf(s) === 0); + const keyText = key?.text.trim() ?? ''; + if (/^[a-z(]/.test(keyText)) return false; + return cols.some((c) => c > 0 && current![c]!.length > 0); + }; + + for (const line of run) { + if (startsRow(line)) { + if (current) grid.push(current); + current = Array.from({ length: nCols }, () => ''); + } + for (const seg of line.segs) { + appendCell(current!, colOf(seg), seg.text); used.push(...seg.indices); } - grid.push(cells); } + if (current) grid.push(current); + return { grid, used }; +} +function finishAlignedTable( + run: RowCand[], + colXs: number[], + page: number, + pageBox: { x: number; y: number; x2: number; y2: number } | undefined, + edges?: number[], +): DetectedTable | undefined { + if (colXs.length < MIN_COLUMNS) return undefined; + const colEdges = edges ?? edgesFromCenters(run, colXs); + const { grid, used } = foldRows(run, colXs, colEdges); + if (grid.length < MIN_ROWS || grid.length > MAX_TABLE_ROWS) return undefined; + if (grid[0]!.length < MIN_COLUMNS) return undefined; if (!acceptGrid(grid)) return undefined; if (looksLikePageColumns(run, grid)) return undefined; if (!acceptBorderless(grid, run, pageBox)) return undefined; @@ -466,6 +625,16 @@ function buildAlignedTable( }; } +function buildAlignedTable( + run: RowCand[], + page: number, + pageBox: { x: number; y: number; x2: number; y2: number } | undefined, +): DetectedTable | undefined { + const cols = inferColumns(run); + if (!cols) return undefined; + return finishAlignedTable(run, cols.colXs, page, pageBox, cols.edges); +} + function looksLikePageColumns(run: RowCand[], grid: string[][]): boolean { if (grid[0]!.length !== 2) return false; if (run.some((r) => r.segs.length >= 3)) return false; @@ -506,7 +675,7 @@ function acceptBorderless( const avgWords = words / Math.max(cells.length, 1); if (avgWords > 4) return false; } - if (pageBox) { + if (pageBox && nCols <= 2) { const x1 = Math.min(...run.flatMap((r) => r.segs.map((s) => s.x))); const x2 = Math.max(...run.flatMap((r) => r.segs.map((s) => s.x2))); const y1 = Math.max(...run.map((r) => r.y)); diff --git a/packages/pdf/test/tables.test.ts b/packages/pdf/test/tables.test.ts index c17488b..b3f5a11 100644 --- a/packages/pdf/test/tables.test.ts +++ b/packages/pdf/test/tables.test.ts @@ -156,6 +156,165 @@ describe('ruled table detection', () => { expect(detectTables(items, [], [])).toHaveLength(0); }); + it('keeps a wrapped cell in one ruled row when verticals are short', () => { + const items = [ + { text: 'Law', x: 30, y: 186, width: 24, page: 1 }, + { text: 'Union/State', x: 90, y: 192, width: 50, page: 1 }, + { text: 'Imprisonment', x: 160, y: 192, width: 50, page: 1 }, + { text: 'rule', x: 100, y: 176, width: 22, page: 1 }, + { text: 'clauses', x: 170, y: 176, width: 32, page: 1 }, + { text: 'Arms Act, 1959', x: 24, y: 158, width: 50, page: 1 }, + { text: 'Union', x: 96, y: 158, width: 28, page: 1 }, + { text: '152', x: 168, y: 158, width: 20, page: 1 }, + { text: 'Food Safety Act', x: 24, y: 130, width: 52, page: 1 }, + { text: 'Licensing Rules', x: 24, y: 114, width: 52, page: 1 }, + { text: 'Union', x: 96, y: 122, width: 28, page: 1 }, + { text: '123', x: 168, y: 122, width: 20, page: 1 }, + { text: 'Regulations, 2011', x: 24, y: 98, width: 56, page: 1 }, + ]; + const lines = [ + { x1: 20, y1: 200, x2: 210, y2: 200, page: 1 }, + { x1: 20, y1: 170, x2: 210, y2: 170, page: 1 }, + { x1: 20, y1: 148, x2: 210, y2: 148, page: 1 }, + { x1: 20, y1: 80, x2: 210, y2: 80, page: 1 }, + { x1: 20, y1: 170, x2: 20, y2: 200, page: 1 }, + { x1: 80, y1: 170, x2: 80, y2: 200, page: 1 }, + { x1: 150, y1: 170, x2: 150, y2: 200, page: 1 }, + { x1: 210, y1: 170, x2: 210, y2: 200, page: 1 }, + { x1: 20, y1: 148, x2: 20, y2: 168, page: 1 }, + { x1: 80, y1: 148, x2: 80, y2: 168, page: 1 }, + { x1: 150, y1: 148, x2: 150, y2: 168, page: 1 }, + { x1: 210, y1: 148, x2: 210, y2: 168, page: 1 }, + { x1: 20, y1: 80, x2: 20, y2: 148, page: 1 }, + { x1: 80, y1: 80, x2: 80, y2: 148, page: 1 }, + { x1: 150, y1: 80, x2: 150, y2: 148, page: 1 }, + { x1: 210, y1: 80, x2: 210, y2: 148, page: 1 }, + ]; + const tables = detectTables(items, lines, []); + expect(tables).toHaveLength(1); + const md = tables[0]!.markdown; + expect(md).toMatch(/\|Law\|Union\/State rule\|Imprisonment clauses\|/); + expect(md).toMatch(/\|Arms Act, 1959\|Union\|152\|/); + expect(md).toMatch(/\|Food Safety Act Licensing Rules Regulations, 2011\|Union\|123\|/); + expect(md.match(/^\|/gm)).toHaveLength(4); + }); + + it('keeps wrapped org names and the header inside one borderless table', () => { + const items = [ + { text: 'No.', x: 22, y: 120, width: 16, page: 1 }, + { text: 'Name of organization', x: 50, y: 120, width: 80, page: 1 }, + { text: 'Number of accredited', x: 160, y: 120, width: 70, page: 1 }, + { text: 'observers', x: 172, y: 108, width: 40, page: 1 }, + { text: '1', x: 24, y: 92, width: 8, page: 1 }, + { text: 'Union of Youth Federations', x: 50, y: 92, width: 90, page: 1 }, + { text: '17,266', x: 176, y: 92, width: 30, page: 1 }, + { text: '(UYFC)', x: 50, y: 80, width: 32, page: 1 }, + { text: '2', x: 24, y: 64, width: 8, page: 1 }, + { text: 'Cambodian Women for Peace', x: 50, y: 64, width: 90, page: 1 }, + { text: '9,835', x: 178, y: 64, width: 26, page: 1 }, + { text: 'Development', x: 50, y: 52, width: 48, page: 1 }, + ]; + const tables = detectTables(items, [], []); + expect(tables).toHaveLength(1); + const md = tables[0]!.markdown; + expect(md).toContain('|No.|Name of organization|Number of accredited observers|'); + expect(md).toContain('|1|Union of Youth Federations (UYFC)|17,266|'); + expect(md).toContain('|2|Cambodian Women for Peace Development|9,835|'); + expect(md.match(/^\|/gm)).toHaveLength(4); + }); + + it('detects a two-column table when most value cells wrap', () => { + const items = [ + { text: 'Field', x: 20, y: 240, width: 30, page: 1 }, + { text: 'Value', x: 80, y: 240, width: 30, page: 1 }, + { text: 'A', x: 20, y: 220, width: 10, page: 1 }, + { text: 'One', x: 80, y: 220, width: 18, page: 1 }, + { text: 'Two', x: 80, y: 206, width: 18, page: 1 }, + { text: 'Three', x: 80, y: 192, width: 24, page: 1 }, + { text: 'B', x: 20, y: 172, width: 10, page: 1 }, + { text: 'Four', x: 80, y: 172, width: 22, page: 1 }, + { text: 'Five', x: 80, y: 158, width: 22, page: 1 }, + { text: 'Six', x: 80, y: 144, width: 18, page: 1 }, + { text: 'C', x: 20, y: 124, width: 10, page: 1 }, + { text: 'Seven', x: 80, y: 124, width: 26, page: 1 }, + { text: 'Eight', x: 80, y: 110, width: 24, page: 1 }, + { text: 'Nine', x: 80, y: 96, width: 22, page: 1 }, + ]; + const tables = detectTables(items, [], []); + expect(tables).toHaveLength(1); + const md = tables[0]!.markdown; + expect(md).toContain('|Field|Value|'); + expect(md).toContain('|A|One Two Three|'); + expect(md).toContain('|B|Four Five Six|'); + expect(md).toContain('|C|Seven Eight Nine|'); + expect(md.match(/^\|/gm)).toHaveLength(5); + }); + + it('folds a short capitalized wrap into the current cell', () => { + const items = [ + { text: 'No.', x: 22, y: 120, width: 16, page: 1 }, + { text: 'Name', x: 50, y: 120, width: 30, page: 1 }, + { text: 'Count', x: 160, y: 120, width: 30, page: 1 }, + { text: '1', x: 24, y: 92, width: 8, page: 1 }, + { text: 'Acme Corp', x: 50, y: 92, width: 50, page: 1 }, + { text: '12', x: 176, y: 92, width: 14, page: 1 }, + { text: 'Inc', x: 50, y: 80, width: 16, page: 1 }, + { text: '2', x: 24, y: 64, width: 8, page: 1 }, + { text: 'New', x: 50, y: 64, width: 20, page: 1 }, + { text: '9', x: 178, y: 64, width: 10, page: 1 }, + { text: 'York', x: 50, y: 52, width: 24, page: 1 }, + ]; + const tables = detectTables(items, [], []); + expect(tables).toHaveLength(1); + const md = tables[0]!.markdown; + expect(md).toContain('|No.|Name|Count|'); + expect(md).toContain('|1|Acme Corp Inc|12|'); + expect(md).toContain('|2|New York|9|'); + expect(md.match(/^\|/gm)).toHaveLength(4); + }); + + it('keeps sparse Y/N cells in their columns across wrapped restriction text', () => { + const items = [ + { text: 'Jurisdiction', x: 20, y: 214, width: 50, page: 1 }, + { text: 'GATS', x: 90, y: 214, width: 28, page: 1 }, + { text: 'Permitted', x: 140, y: 214, width: 40, page: 1 }, + { text: 'Restrictions', x: 200, y: 214, width: 50, page: 1 }, + { text: 'Reporting', x: 320, y: 214, width: 40, page: 1 }, + { text: '(1994)', x: 90, y: 200, width: 28, page: 1 }, + { text: 'Requirements', x: 320, y: 200, width: 50, page: 1 }, + { text: 'Argentina', x: 20, y: 180, width: 44, page: 1 }, + { text: 'Y', x: 98, y: 180, width: 10, page: 1 }, + { text: 'Y', x: 150, y: 180, width: 10, page: 1 }, + { text: 'Prohibition on ownership of', x: 200, y: 180, width: 90, page: 1 }, + { text: 'property that borders water.', x: 200, y: 166, width: 90, page: 1 }, + { text: 'Australia', x: 20, y: 140, width: 44, page: 1 }, + { text: 'N', x: 98, y: 140, width: 10, page: 1 }, + { text: 'Y', x: 150, y: 140, width: 10, page: 1 }, + { text: 'Approval is needed from the', x: 200, y: 140, width: 90, page: 1 }, + { text: 'Treasurer for land purchases.', x: 200, y: 126, width: 90, page: 1 }, + { text: 'Must report acquisitions.', x: 320, y: 140, width: 70, page: 1 }, + { text: 'to the agency.', x: 320, y: 126, width: 50, page: 1 }, + { text: 'Belgium', x: 20, y: 100, width: 40, page: 1 }, + { text: 'N', x: 98, y: 100, width: 10, page: 1 }, + { text: 'Y', x: 150, y: 100, width: 10, page: 1 }, + { text: 'None.', x: 200, y: 100, width: 24, page: 1 }, + ]; + const tables = detectTables(items, [], []); + expect(tables).toHaveLength(1); + const md = tables[0]!.markdown; + expect(md).toContain( + '|Jurisdiction|GATS (1994)|Permitted|Restrictions|Reporting Requirements|', + ); + expect(md).toContain( + '|Argentina|Y|Y|Prohibition on ownership of property that borders water.||', + ); + expect(md).toContain( + '|Australia|N|Y|Approval is needed from the Treasurer for land purchases.|Must report acquisitions. to the agency.|', + ); + expect(md).toContain('|Belgium|N|Y|None.||'); + expect(md.match(/^\|/gm)).toHaveLength(5); + }); + it('detects two separate ruled grids on one page', () => { const items = [ { text: 'Name', x: 22, y: 182, width: 30, page: 1 },