From 26c3da03752b9b431bc715ae13fc5c0eb81766f8 Mon Sep 17 00:00:00 2001 From: caichuanwang Date: Wed, 5 Aug 2026 13:22:27 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E5=80=9F=E9=89=B4=20pdf-inspector=20?= =?UTF-8?q?=E6=80=9D=E8=B7=AF=E8=A1=A5=E9=BD=90=E5=8E=9F=E7=94=9F=20PDF=20?= =?UTF-8?q?=E5=B8=83=E5=B1=80=E3=80=81=E8=AF=AD=E4=B9=89=E4=B8=8E=E8=A1=A8?= =?UTF-8?q?=E6=A0=BC=E8=B4=A8=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PDF 原生路径此前逐词输出且仅按 (top,left) 排序，字体元数据采集后即丢失， 导致双栏交错、段落碎片化、标题/列表/代码不可表达，页眉页脚与断词也原样泄漏。 - extract.py: 词级候选 + 行重建 + 双栏/跨栏阅读顺序，行分割与列聚类有界 - semantics.py: 文档级正文基准字号、标题/列表/代码块/段落语义、保守断词、 重复页眉页脚与页码过滤；代码围栏动态加长防逃逸 - analyze.py: 移除不可控 pdfplumber text/text 回退，改为 ≤500 词的有界本地 无边框表格启发式（3 行 ×2 列、数字占比 ≥40%） - extract.py: 拒绝一维伪表格；文本不同的嵌套表不再被去重吞掉； Unicode 非字符/代理项纳入编码质量检测 - merge.py: 表格/视觉区域按词中心点所有权，hybrid 元素按所属栏插入， 保持列阅读顺序；所有权裁决后再重建行 - models.py: NativeTextCandidate 携带 font_size/font_name 并贯通 wire - parser.py: _has_semantic_blocks 识别标题/列表/代码块，避免误报无内容 Tested: 全量 pytest 634 passed/9 skipped；ruff check/format、ty check、 git diff --check 全部通过；新增真实合成 PDF 反例（对齐段落不误判表格、 无边框数字表格可恢复）及双栏/跨栏/断词/页码/Unicode 边界测试 Scope-risk: 原生路由 PDF 的输出从逐词 TextBlock 变为语义块，依赖视觉的 hybrid/full-vision 页面输出不变；跨栏表格与视觉插入顺序为新启发式 Not-tested: 私有验收语料与真实多页复杂版式未在本次验证；pdf-inspector 预分类层与 OCR 回退路由暂未引入 --- src/opendocs/parsers/pdf/analyze.py | 7 +- src/opendocs/parsers/pdf/extract.py | 370 ++++++++++++++++++++++++-- src/opendocs/parsers/pdf/merge.py | 223 ++++++++++++---- src/opendocs/parsers/pdf/models.py | 6 + src/opendocs/parsers/pdf/parser.py | 11 + src/opendocs/parsers/pdf/semantics.py | 281 +++++++++++++++++++ tests/test_pdf_analyze.py | 136 ++++++++++ tests/test_pdf_extract.py | 227 +++++++++++++++- tests/test_pdf_merge.py | 280 ++++++++++++++++++- tests/test_pdf_parser.py | 25 +- 10 files changed, 1481 insertions(+), 85 deletions(-) create mode 100644 src/opendocs/parsers/pdf/semantics.py diff --git a/src/opendocs/parsers/pdf/analyze.py b/src/opendocs/parsers/pdf/analyze.py index 71e7da9..0716220 100644 --- a/src/opendocs/parsers/pdf/analyze.py +++ b/src/opendocs/parsers/pdf/analyze.py @@ -17,6 +17,7 @@ bbox_contains_center, build_native_candidates, build_table_candidate, + detect_heuristic_table, is_table_valid, is_text_reliable, measure_text_quality, @@ -308,7 +309,7 @@ def _wire_estimate( object_bbox_count: int = 0, ) -> int: estimate = sum( - _WIRE_WORD_OVERHEAD + len(word.text) * 8 + len(word.font_name or "") * 4 for word in words + _WIRE_WORD_OVERHEAD + len(word.text) * 8 + len(word.font_name or "") * 8 for word in words ) for table in tables: estimate += _WIRE_TABLE_OVERHEAD @@ -352,6 +353,10 @@ def _analyze_page(page: Any, page_number: int) -> PageFacts: all_tables, table_failed = _tables(page, words, geometry) _enforce_wire_budget(words, all_tables) accepted_tables = select_canonical_tables(all_tables) + if not accepted_tables: + heuristic_table = detect_heuristic_table(words) + if heuristic_table is not None: + accepted_tables = (heuristic_table,) native_candidates = build_native_candidates(words, accepted_tables) all_text = " ".join(word.text for word in words) quality = measure_text_quality(all_text) diff --git a/src/opendocs/parsers/pdf/extract.py b/src/opendocs/parsers/pdf/extract.py index 0ec4e05..48cb09a 100644 --- a/src/opendocs/parsers/pdf/extract.py +++ b/src/opendocs/parsers/pdf/extract.py @@ -22,7 +22,24 @@ _TABLE_UNKNOWN_RATIO_MAX = 0.40 _TABLE_TEXT_CAPTURE_RATIO_MIN = 0.80 _TABLE_DUPLICATE_IOU_MIN = 0.80 +_TABLE_NESTED_TEXT_SIMILARITY_MIN = 0.80 _CID_PATTERN = re.compile(r"\(cid:\d+\)") +_CJK_RANGES = ( + (0x2E80, 0x9FFF), + (0xF900, 0xFAFF), + (0x20000, 0x2FA1F), +) +_LINE_CENTER_TOLERANCE = 0.55 +_LINE_SPLIT_GAP_FACTOR = 2.5 +_MAX_ACTIVE_LINES = 64 +_COLUMN_GAP_MIN = 0.12 +_COLUMN_OVERLAP_MIN = 0.20 +_WIDE_BLOCK_WIDTH_MIN = 0.70 +_HEURISTIC_TABLE_MAX_WORDS = 500 +_HEURISTIC_TABLE_MIN_ROWS = 3 +_HEURISTIC_TABLE_MIN_COLUMNS = 2 +_HEURISTIC_TABLE_COLUMN_TOLERANCE = 0.025 +_HEURISTIC_TABLE_NUMERIC_RATIO_MIN = 0.40 def bbox_area(bbox: BBox) -> float: @@ -88,8 +105,14 @@ def measure_text_quality(text: str) -> TextQuality: denominator = max(len(characters), 1) bad_indexes: set[int] = set() for index, character in enumerate(text): - if character == "�" or ( - unicodedata.category(character) == "Cc" and character not in "\t\n\r" + category = unicodedata.category(character) + value = ord(character) + if ( + character == "�" + or (category == "Cc" and character not in "\t\n\r") + or category == "Cs" + or value & 0xFFFF in {0xFFFE, 0xFFFF} + or 0xFDD0 <= value <= 0xFDEF ): bad_indexes.add(index) for match in _CID_PATTERN.finditer(text): @@ -185,7 +208,8 @@ def is_table_valid(candidate: NativeTableCandidate) -> bool: return bool( candidate.grid and widths - and min(widths) > 0 + and min(widths) >= 2 + and len(candidate.grid) >= 2 and len(widths) == 1 and candidate.nonempty_ratio >= _TABLE_NONEMPTY_RATIO_MIN and candidate.unknown_ratio <= _TABLE_UNKNOWN_RATIO_MAX @@ -193,6 +217,37 @@ def is_table_valid(candidate: NativeTableCandidate) -> bool: ) +def _table_text_similarity(left: NativeTableCandidate, right: NativeTableCandidate) -> float: + left_text = Counter( + character + for row in left.grid + for cell in row + if cell is not None + for character in cell + if not character.isspace() + ) + right_text = Counter( + character + for row in right.grid + for cell in row + if cell is not None + for character in cell + if not character.isspace() + ) + shared = sum((left_text & right_text).values()) + return shared / max(min(sum(left_text.values()), sum(right_text.values())), 1) + + +def _tables_are_duplicates( + left: NativeTableCandidate, + right: NativeTableCandidate, +) -> bool: + if intersection_over_union(left.bbox, right.bbox) >= _TABLE_DUPLICATE_IOU_MIN: + return True + nested = bbox_contains(left.bbox, right.bbox) or bbox_contains(right.bbox, left.bbox) + return nested and _table_text_similarity(left, right) >= _TABLE_NESTED_TEXT_SIMILARITY_MIN + + def select_canonical_tables( candidates: Iterable[NativeTableCandidate], ) -> tuple[NativeTableCandidate, ...]: @@ -208,12 +263,7 @@ def select_canonical_tables( ) accepted: list[NativeTableCandidate] = [] for candidate in ordered: - if any( - bbox_contains(existing.bbox, candidate.bbox) - or bbox_contains(candidate.bbox, existing.bbox) - or intersection_over_union(existing.bbox, candidate.bbox) >= _TABLE_DUPLICATE_IOU_MIN - for existing in accepted - ): + if any(_tables_are_duplicates(existing, candidate) for existing in accepted): continue accepted.append(candidate) return tuple( @@ -228,6 +278,260 @@ def _line_key(word: PdfWord) -> tuple[float, float, int]: return word.bbox.top, word.bbox.left, word.source_index +def _is_cjk(character: str) -> bool: + value = ord(character) + return any(start <= value <= end for start, end in _CJK_RANGES) + + +def _word_separator(left: PdfWord, right: PdfWord) -> str: + if not left.text or not right.text: + return "" + if _is_cjk(left.text[-1]) or _is_cjk(right.text[0]): + return "" + if right.text[0] in ",.;:!?)]}%" or left.text[-1] in "([{$/": + return "" + return " " + + +def _same_line(left: PdfWord, right: PdfWord) -> bool: + left_center = (left.bbox.top + left.bbox.bottom) / 2 + right_center = (right.bbox.top + right.bbox.bottom) / 2 + height = min(left.bbox.bottom - left.bbox.top, right.bbox.bottom - right.bbox.top) + return abs(left_center - right_center) <= max(height * _LINE_CENTER_TOLERANCE, 0.002) + + +def _line_word_groups(words: Sequence[PdfWord]) -> list[list[PdfWord]]: + lines: list[list[PdfWord]] = [] + for word in sorted(words, key=_line_key): + matched = False + for line in reversed(lines[-_MAX_ACTIVE_LINES:]): + if _same_line(line[0], word): + line.append(word) + matched = True + break + if word.bbox.top - line[0].bbox.bottom > 0.02: + break + if not matched: + lines.append([word]) + return [sorted(line, key=lambda item: (item.bbox.left, item.source_index)) for line in lines] + + +def _word_groups(words: Sequence[PdfWord]) -> list[list[PdfWord]]: + groups: list[list[PdfWord]] = [] + for ordered in _line_word_groups(words): + current: list[PdfWord] = [] + for word in ordered: + if current: + previous = current[-1] + typical_height = max(previous.bbox.bottom - previous.bbox.top, 0.001) + if word.bbox.left - previous.bbox.right > typical_height * _LINE_SPLIT_GAP_FACTOR: + groups.append(current) + current = [] + current.append(word) + if current: + groups.append(current) + return groups + + +def _line_candidate(words: Sequence[PdfWord]) -> NativeTextCandidate: + ordered = sorted(words, key=lambda word: (word.bbox.left, word.source_index)) + text = ordered[0].text + for left, right in pairwise(ordered): + text += _word_separator(left, right) + right.text + font_sizes = [word.font_size for word in ordered if word.font_size is not None] + font_names = [word.font_name for word in ordered if word.font_name] + font_size = sum(font_sizes) / len(font_sizes) if font_sizes else None + font_name = Counter(font_names).most_common(1)[0][0] if font_names else None + return NativeTextCandidate( + text=text, + bbox=BBox( + min(word.bbox.left for word in ordered), + min(word.bbox.top for word in ordered), + max(word.bbox.right for word in ordered), + max(word.bbox.bottom for word in ordered), + ), + source_index=min(word.source_index for word in ordered), + font_size=font_size, + font_name=font_name, + ) + + +def _column_order_narrow( + candidates: Sequence[NativeTextCandidate], +) -> list[NativeTextCandidate]: + if len(candidates) < 2: + return list(candidates) + + def fallback() -> list[NativeTextCandidate]: + return sorted( + candidates, + key=lambda item: (item.bbox.top, item.bbox.left, item.source_index), + ) + + ordered = sorted(candidates, key=lambda item: (item.bbox.left, item.bbox.top)) + columns: list[tuple[float, float, list[NativeTextCandidate]]] = [] + for candidate in ordered: + for index, (column_left, column_right, column) in enumerate(columns): + overlap = max( + 0.0, + min(column_right, candidate.bbox.right) - max(column_left, candidate.bbox.left), + ) + narrower = min(column_right - column_left, candidate.bbox.right - candidate.bbox.left) + if overlap / max(narrower, 0.001) >= _COLUMN_OVERLAP_MIN: + column.append(candidate) + columns[index] = ( + min(column_left, candidate.bbox.left), + max(column_right, candidate.bbox.right), + column, + ) + break + else: + columns.append((candidate.bbox.left, candidate.bbox.right, [candidate])) + if len(columns) > 2: + return fallback() + + left_right = sorted(columns, key=lambda item: item[0]) + if len(left_right) != 2 or left_right[1][0] - left_right[0][1] < _COLUMN_GAP_MIN: + return fallback() + return [ + item + for _, _, column in left_right + for item in sorted( + column, + key=lambda value: (value.bbox.top, value.bbox.left, value.source_index), + ) + ] + + +def _column_order(candidates: Sequence[NativeTextCandidate]) -> list[NativeTextCandidate]: + if len(candidates) < 2: + return list(candidates) + + wide = sorted( + ( + candidate + for candidate in candidates + if candidate.bbox.right - candidate.bbox.left >= _WIDE_BLOCK_WIDTH_MIN + ), + key=lambda item: (item.bbox.top, item.bbox.left, item.source_index), + ) + if wide: + narrow = [candidate for candidate in candidates if candidate not in wide] + result: list[NativeTextCandidate] = [] + for spanning in wide: + before = [candidate for candidate in narrow if candidate.bbox.top < spanning.bbox.top] + result.extend(_column_order_narrow(before)) + narrow = [candidate for candidate in narrow if candidate not in before] + result.append(spanning) + result.extend(_column_order_narrow(narrow)) + return result + return _column_order_narrow(candidates) + + +def _numeric_cell(text: str) -> bool: + normalized = text.strip().replace(",", "").replace("%", "") + normalized = normalized.removeprefix("$").removeprefix("¥").removeprefix("€") + normalized = normalized.strip("()") + try: + float(normalized) + except ValueError: + return False + return True + + +def detect_heuristic_table(words: Sequence[PdfWord]) -> NativeTableCandidate | None: + if not words or len(words) > _HEURISTIC_TABLE_MAX_WORDS: + return None + lines = [line for line in _line_word_groups(words) if len(line) >= 2] + if len(lines) < _HEURISTIC_TABLE_MIN_ROWS: + return None + + clusters: list[list[float]] = [] + for word in sorted(words, key=lambda item: item.bbox.left): + for cluster in clusters: + anchor = sum(cluster) / len(cluster) + if abs(word.bbox.left - anchor) <= _HEURISTIC_TABLE_COLUMN_TOLERANCE: + cluster.append(word.bbox.left) + break + else: + clusters.append([word.bbox.left]) + anchors = sorted( + sum(cluster) / len(cluster) + for cluster in clusters + if len(cluster) >= _HEURISTIC_TABLE_MIN_ROWS + ) + if not _HEURISTIC_TABLE_MIN_COLUMNS <= len(anchors) <= 8: + return None + + grid: list[list[str | None]] = [] + for line in lines: + row: list[list[str]] = [[] for _ in anchors] + for word in line: + distances = [abs(word.bbox.left - anchor) for anchor in anchors] + column = min(range(len(anchors)), key=distances.__getitem__) + if distances[column] <= _HEURISTIC_TABLE_COLUMN_TOLERANCE: + row[column].append(word.text) + cells = [" ".join(parts) if parts else None for parts in row] + if sum(bool(cell) for cell in cells) >= _HEURISTIC_TABLE_MIN_COLUMNS: + grid.append(cells) + if len(grid) < _HEURISTIC_TABLE_MIN_ROWS: + return None + + data_cells = [cell for row in grid[1:] for cell in row if cell] + numeric_ratio = sum(_numeric_cell(cell) for cell in data_cells) / max(len(data_cells), 1) + if numeric_ratio < _HEURISTIC_TABLE_NUMERIC_RATIO_MIN: + return None + + bbox = BBox( + min(word.bbox.left for word in words), + min(word.bbox.top for word in words), + max(word.bbox.right for word in words), + max(word.bbox.bottom for word in words), + ) + candidate = build_table_candidate( + bbox=bbox, + raw_grid=grid, + words=words, + source_index=0, + ) + return candidate if is_table_valid(candidate) else None + + +def build_native_text_lines( + candidates: Sequence[NativeTextCandidate], +) -> tuple[NativeTextCandidate, ...]: + words = tuple( + PdfWord( + candidate.text, + candidate.bbox, + candidate.source_index, + candidate.font_size, + candidate.font_name, + ) + for candidate in candidates + ) + return tuple(_column_order([_line_candidate(group) for group in _word_groups(words)])) + + +def _ordered_words(words: Sequence[PdfWord]) -> list[PdfWord]: + groups = _word_groups(words) + group_by_source = { + min(word.source_index for word in group): sorted( + group, + key=lambda word: (word.bbox.left, word.source_index), + ) + for group in groups + } + ordered_lines = _column_order([_line_candidate(group) for group in groups]) + return [word for line in ordered_lines for word in group_by_source.get(line.source_index, ())] + + +def _same_layout_column(left: BBox, right: BBox) -> bool: + overlap = max(0.0, min(left.right, right.right) - max(left.left, right.left)) + narrower = min(left.right - left.left, right.right - right.left) + return overlap / max(narrower, 0.001) >= _COLUMN_OVERLAP_MIN + + def build_native_candidates( words: Sequence[PdfWord], tables: Sequence[NativeTableCandidate] ) -> tuple[NativeCandidate, ...]: @@ -235,20 +539,40 @@ def build_native_candidates( word for word in words if not any(bbox_contains_center(table.bbox, word.bbox) for table in tables) + and word.text.strip() ] - candidates: list[NativeCandidate] = [ - NativeTextCandidate(word.text, word.bbox, word.source_index) - for word in sorted(owned_words, key=_line_key) - if word.text.strip() + ordered: list[NativeCandidate] = [ + NativeTextCandidate( + word.text, + word.bbox, + word.source_index, + word.font_size, + word.font_name, + ) + for word in _ordered_words(owned_words) ] - candidates.extend(tables) - return tuple( - sorted( - candidates, - key=lambda candidate: ( - candidate.bbox.top, - candidate.bbox.left, - candidate.source_index, - ), + for table in sorted( + tables, + key=lambda candidate: ( + candidate.bbox.top, + candidate.bbox.left, + candidate.source_index, + ), + ): + spanning = table.bbox.right - table.bbox.left >= _WIDE_BLOCK_WIDTH_MIN + if spanning: + before = [candidate for candidate in ordered if candidate.bbox.bottom <= table.bbox.top] + after = [candidate for candidate in ordered if candidate not in before] + ordered = [*before, table, *after] + continue + same_column = [ + index + for index, candidate in enumerate(ordered) + if _same_layout_column(candidate.bbox, table.bbox) + ] + insert_at = next( + (index for index in same_column if ordered[index].bbox.top >= table.bbox.top), + same_column[-1] + 1 if same_column else len(ordered), ) - ) + ordered.insert(insert_at, table) + return tuple(ordered) diff --git a/src/opendocs/parsers/pdf/merge.py b/src/opendocs/parsers/pdf/merge.py index cce939a..656d7ce 100644 --- a/src/opendocs/parsers/pdf/merge.py +++ b/src/opendocs/parsers/pdf/merge.py @@ -3,15 +3,8 @@ import re from dataclasses import dataclass -from opendocs._models import ( - BBox, - Block, - PageBreakBlock, - TableBlock, - TextBlock, - WarningRecord, -) -from opendocs.parsers.pdf.extract import intersection_area +from opendocs._models import BBox, Block, PageBreakBlock, TableBlock, TextBlock, WarningRecord +from opendocs.parsers.pdf.extract import bbox_contains_center, build_native_text_lines from opendocs.parsers.pdf.models import ( NativeCandidate, NativeTableCandidate, @@ -19,6 +12,7 @@ PageFacts, PageRoute, ) +from opendocs.parsers.pdf.semantics import body_font_size, edge_suppressions, structured_native_run from opendocs.vision.base import VisionElement, VisionTableElement, VisionTextElement PAGE_NORMALIZED_V1 = "page-normalized-v1" @@ -49,10 +43,6 @@ def _visual_bbox(element: VisionElement, *, required: bool) -> BBox | None: return bbox.require_normalized("visual bbox") -def _intersects(left: BBox, right: BBox) -> bool: - return intersection_area(left, right) > 0 - - def _native_tables_own_text(candidates: tuple[NativeCandidate, ...]) -> tuple[NativeCandidate, ...]: table_boxes = tuple( candidate.bbox for candidate in candidates if isinstance(candidate, NativeTableCandidate) @@ -62,7 +52,7 @@ def _native_tables_own_text(candidates: tuple[NativeCandidate, ...]) -> tuple[Na for candidate in candidates if not ( isinstance(candidate, NativeTextCandidate) - and any(_intersects(candidate.bbox, table_bbox) for table_bbox in table_boxes) + and any(bbox_contains_center(table_bbox, candidate.bbox) for table_bbox in table_boxes) ) ) @@ -79,20 +69,11 @@ def _visual_tables_own_text(elements: tuple[VisionElement, ...]) -> tuple[Vision if not ( isinstance(element, VisionTextElement) and element.bbox is not None - and any(_intersects(element.bbox, table_bbox) for table_bbox in table_boxes) + and any(bbox_contains_center(table_bbox, element.bbox) for table_bbox in table_boxes) ) ) -def _candidate_key(candidate: NativeCandidate) -> tuple[float, float, int, int]: - return ( - candidate.bbox.top, - candidate.bbox.left, - candidate.source_index, - 0 if isinstance(candidate, NativeTableCandidate) else 1, - ) - - def _element_key(element: VisionElement) -> tuple[float, float, int, int]: bbox = element.bbox return ( @@ -115,37 +96,158 @@ def _visual_block(element: VisionElement) -> Block: return TextBlock(element.text) -def _page_blocks(page: PageFacts, visual: PageVisionResult | None) -> tuple[Block, ...]: - native_candidates = _native_tables_own_text(page.native_candidates) - if visual is None: - return tuple( - _native_block(candidate) for candidate in sorted(native_candidates, key=_candidate_key) +def _item_bbox(item: NativeCandidate | VisionElement) -> BBox | None: + return item.bbox + + +def _layout_columns( + items: list[NativeCandidate | VisionElement], +) -> list[list[int]] | None: + columns: list[tuple[float, float, list[int]]] = [] + for index, item in enumerate(items): + bbox = _item_bbox(item) + if bbox is None or bbox.right - bbox.left >= 0.70: + continue + for column_index, (left, right, indexes) in enumerate(columns): + overlap = max(0.0, min(right, bbox.right) - max(left, bbox.left)) + narrower = min(right - left, bbox.right - bbox.left) + if overlap / max(narrower, 0.001) >= 0.20: + indexes.append(index) + columns[column_index] = (min(left, bbox.left), max(right, bbox.right), indexes) + break + else: + columns.append((bbox.left, bbox.right, [index])) + if len(columns) > 2: + return None + columns.sort(key=lambda column: column[0]) + if ( + len(columns) != 2 + or sum(len(column[2]) for column in columns) < 3 + or columns[1][0] - columns[0][1] < 0.12 + ): + return None + return [column[2] for column in columns] + + +def _insert_visual_item( + items: list[NativeCandidate | VisionElement], + element: VisionElement, + bbox: BBox, +) -> list[NativeCandidate | VisionElement]: + if bbox.right - bbox.left >= 0.70: + before = [ + item + for item in items + if (item_bbox := _item_bbox(item)) is not None and item_bbox.bottom <= bbox.top + ] + after = [item for item in items if item not in before] + return [*before, element, *after] + + columns = _layout_columns(items) + if columns is not None: + overlaps = [] + for indexes in columns: + column_boxes = [ + item_bbox + for index in indexes + if (item_bbox := _item_bbox(items[index])) is not None + ] + left = min(box.left for box in column_boxes) + right = max(box.right for box in column_boxes) + overlaps.append(max(0.0, min(right, bbox.right) - max(left, bbox.left))) + target = max(range(len(columns)), key=overlaps.__getitem__) + if overlaps[target] > 0: + target_indexes = columns[target] + insert_at = next( + ( + index + for index in target_indexes + if (item_bbox := _item_bbox(items[index])) is not None + and item_bbox.top >= bbox.top + ), + target_indexes[-1] + 1, + ) + items.insert(insert_at, element) + return items + + insert_at = next( + ( + index + for index, item in enumerate(items) + if (item_bbox := _item_bbox(item)) is not None and item_bbox.top >= bbox.top + ), + len(items), + ) + items.insert(insert_at, element) + return items + + +def _page_blocks( + page: PageFacts, + visual: PageVisionResult | None, + *, + suppressed: set[tuple[int, int]], + body_size: float | None, +) -> tuple[Block, ...]: + native_candidates = tuple( + candidate + for candidate in _native_tables_own_text(page.native_candidates) + if not ( + isinstance(candidate, NativeTextCandidate) + and (page.page_number, candidate.source_index) in suppressed ) - - elements = _visual_tables_own_text(visual.elements) - if visual.route is PageRoute.FULL_VISION: + ) + if visual is not None and visual.route is PageRoute.FULL_VISION: + elements = _visual_tables_own_text(visual.elements) return tuple(_visual_block(element) for element in sorted(elements, key=_element_key)) - if visual.route is not PageRoute.HYBRID: - return tuple( - _native_block(candidate) for candidate in sorted(native_candidates, key=_candidate_key) - ) - if visual.coordinate_space != PAGE_NORMALIZED_V1: - raise ValueError("hybrid merge requires page-normalized-v1 visual coordinates") - visual_boxes = tuple(_visual_bbox(element, required=True) for element in elements) - native = tuple( - candidate - for candidate in native_candidates - if not any( - visual_bbox is not None and _intersects(candidate.bbox, visual_bbox) - for visual_bbox in visual_boxes + items: list[NativeCandidate | VisionElement] = list(native_candidates) + if visual is not None and visual.route is PageRoute.HYBRID: + if visual.coordinate_space != PAGE_NORMALIZED_V1: + raise ValueError("hybrid merge requires page-normalized-v1 visual coordinates") + elements = _visual_tables_own_text(visual.elements) + visual_boxes = tuple(_visual_bbox(element, required=True) for element in elements) + items = [ + candidate + for candidate in items + if not ( + isinstance(candidate, NativeTextCandidate) + and any( + visual_bbox is not None and bbox_contains_center(visual_bbox, candidate.bbox) + for visual_bbox in visual_boxes + ) + ) + ] + for element in sorted(elements, key=_element_key): + visual_bbox = _visual_bbox(element, required=True) + if visual_bbox is None: + raise AssertionError("required visual bbox disappeared") + items = _insert_visual_item(items, element, visual_bbox) + + blocks: list[Block] = [] + native_run: list[NativeTextCandidate] = [] + + def flush_native() -> None: + if not native_run: + return + blocks.extend( + structured_native_run( + list(build_native_text_lines(native_run)), + body_size=body_size, + ) ) - ) - candidates: list[tuple[float, float, int, int, Block]] = [ - (*_candidate_key(candidate), _native_block(candidate)) for candidate in native - ] - candidates.extend((*_element_key(element), _visual_block(element)) for element in elements) - return tuple(item[-1] for item in sorted(candidates, key=lambda item: item[:-1])) + native_run.clear() + + for item in items: + if isinstance(item, NativeTextCandidate): + native_run.append(item) + continue + flush_native() + blocks.append( + _native_block(item) if isinstance(item, NativeTableCandidate) else _visual_block(item) + ) + flush_native() + return tuple(blocks) _PAGE_WARNING = re.compile(r"^PDF page (\d+):") @@ -168,13 +270,22 @@ def merge_pdf_pages( if len(visual_slots) != len(visual_results) or not set(visual_slots) <= set(page_slots): raise ValueError("PDF visual results must map to unique analyzed pages") + ordered_pages = tuple(page_slots[number] for number in sorted(page_slots)) + suppressed = edge_suppressions(ordered_pages) + body_size = body_font_size(ordered_pages, suppressed) blocks: list[Block] = [] warnings = list(document_warnings) - for page_number in sorted(page_slots): - page = page_slots[page_number] - visual = visual_slots.get(page_number) - blocks.append(PageBreakBlock(page_number)) - blocks.extend(_page_blocks(page, visual)) + for page in ordered_pages: + visual = visual_slots.get(page.page_number) + blocks.append(PageBreakBlock(page.page_number)) + blocks.extend( + _page_blocks( + page, + visual, + suppressed=suppressed, + body_size=body_size, + ) + ) if visual is not None: warnings.extend(visual.warnings) warnings.sort(key=_warning_key) diff --git a/src/opendocs/parsers/pdf/models.py b/src/opendocs/parsers/pdf/models.py index c8a7763..69154ab 100644 --- a/src/opendocs/parsers/pdf/models.py +++ b/src/opendocs/parsers/pdf/models.py @@ -35,6 +35,8 @@ class NativeTextCandidate: text: str bbox: BBox source_index: int + font_size: float | None = None + font_name: str | None = None @dataclass(frozen=True, slots=True) @@ -182,6 +184,8 @@ def candidate_to_wire(candidate: NativeCandidate) -> dict[str, object]: "text": candidate.text, "bbox": bbox_to_wire(candidate.bbox), "source_index": candidate.source_index, + "font_size": candidate.font_size, + "font_name": candidate.font_name, }, } @@ -199,6 +203,8 @@ def candidate_from_wire(value: object) -> NativeCandidate: text=str(payload["text"]), bbox=bbox_from_wire(payload["bbox"]), source_index=int(payload["source_index"]), + font_size=(float(payload["font_size"]) if payload.get("font_size") is not None else None), + font_name=str(payload["font_name"]) if payload.get("font_name") is not None else None, ) diff --git a/src/opendocs/parsers/pdf/parser.py b/src/opendocs/parsers/pdf/parser.py index 94b1125..668e167 100644 --- a/src/opendocs/parsers/pdf/parser.py +++ b/src/opendocs/parsers/pdf/parser.py @@ -10,6 +10,10 @@ from opendocs._models import ( BBox, DocumentType, + HeadingBlock, + ListItemBlock, + MarkdownBlock, + ParagraphBlock, ParsedDocument, TableBlock, TextBlock, @@ -131,6 +135,13 @@ def _has_semantic_blocks(blocks: tuple[object, ...]) -> bool: for block in blocks: if isinstance(block, TextBlock) and block.text.strip(): return True + if isinstance(block, MarkdownBlock) and block.markdown.strip(): + return True + if isinstance(block, ParagraphBlock | HeadingBlock | ListItemBlock) and any( + getattr(inline, "text", "").strip() or getattr(inline, "label", "").strip() + for inline in block.inlines + ): + return True if isinstance(block, TableBlock) and any( cell.strip() for row in block.grid for cell in row ): diff --git a/src/opendocs/parsers/pdf/semantics.py b/src/opendocs/parsers/pdf/semantics.py new file mode 100644 index 0000000..8788dd5 --- /dev/null +++ b/src/opendocs/parsers/pdf/semantics.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +import re + +from opendocs._models import ( + Block, + HeadingBlock, + InlineText, + ListItemBlock, + ListKind, + MarkdownBlock, + ParagraphBlock, + TextBlock, +) +from opendocs.parsers.pdf.models import NativeTextCandidate, PageFacts + +_EDGE_TOP_MAX = 0.08 +_EDGE_BOTTOM_MIN = 0.92 +_EDGE_REPEAT_MIN_PAGES = 3 +_EDGE_REPEAT_PERCENT = 60 +_BULLET_LIST = re.compile(r"^[•◦▪‣\u2043*+-]\s+(.+)$") +_ORDERED_LIST = re.compile(r"^(\d{1,9})[.)]\s+(.+)$") +_PAGE_NUMBER = re.compile(r"^(?:page\s+)?\d+(?:\s+(?:of|/)\s+\d+)?$", re.IGNORECASE) +_MONOSPACE_MARKERS = ("mono", "courier", "consolas", "menlo", "fira code", "jetbrains") +_DEHYPHENATION_SUFFIXES = ( + "able", + "al", + "ation", + "ed", + "er", + "est", + "ible", + "ing", + "ity", + "ive", + "ly", + "ment", + "ness", + "ous", +) + + +def _normalize_edge_text(text: str) -> str: + return " ".join(text.casefold().split()) + + +def edge_suppressions(pages: tuple[PageFacts, ...]) -> set[tuple[int, int]]: + occurrences: dict[tuple[str, str], set[int]] = {} + candidates: list[tuple[PageFacts, NativeTextCandidate, str]] = [] + for page in pages: + for candidate in page.native_candidates: + if not isinstance(candidate, NativeTextCandidate): + continue + edge = ( + "top" + if candidate.bbox.top <= _EDGE_TOP_MAX + else "bottom" + if candidate.bbox.bottom >= _EDGE_BOTTOM_MIN + else "" + ) + if not edge: + continue + normalized = _normalize_edge_text(candidate.text) + if not normalized: + continue + occurrences.setdefault((edge, normalized), set()).add(page.page_number) + candidates.append((page, candidate, edge)) + + required = max( + _EDGE_REPEAT_MIN_PAGES, + (len(pages) * _EDGE_REPEAT_PERCENT + 99) // 100, + ) + suppressed: set[tuple[int, int]] = set() + for page, candidate, edge in candidates: + normalized = _normalize_edge_text(candidate.text) + if len(occurrences[(edge, normalized)]) >= required or ( + edge == "bottom" and _PAGE_NUMBER.fullmatch(normalized) + ): + suppressed.add((page.page_number, candidate.source_index)) + return suppressed + + +def body_font_size( + pages: tuple[PageFacts, ...], + suppressed: set[tuple[int, int]], +) -> float | None: + weights: dict[float, int] = {} + for page in pages: + for candidate in page.native_candidates: + if not isinstance(candidate, NativeTextCandidate): + continue + font_size = candidate.font_size + if ( + font_size is not None + and (page.page_number, candidate.source_index) not in suppressed + and candidate.bbox.top > _EDGE_TOP_MAX + and candidate.bbox.bottom < _EDGE_BOTTOM_MIN + ): + size = float(round(font_size * 2) / 2) + weights[size] = weights.get(size, 0) + len(candidate.text.strip()) + + best_size: float | None = None + best_weight = -1 + for size, weight in weights.items(): + if weight > best_weight or ( + weight == best_weight and (best_size is None or size < best_size) + ): + best_size = size + best_weight = weight + return best_size + + +def _is_monospace(candidate: NativeTextCandidate) -> bool: + name = (candidate.font_name or "").casefold() + return any(marker in name for marker in _MONOSPACE_MARKERS) + + +def _fenced_code(lines: list[str]) -> MarkdownBlock: + longest = max( + (len(match.group(0)) for line in lines for match in re.finditer(r"`+", line)), + default=0, + ) + fence = "`" * max(3, longest + 1) + content = "\n".join(lines) + return MarkdownBlock(f"{fence}\n{content}\n{fence}") + + +def _is_bold(candidate: NativeTextCandidate) -> bool: + name = (candidate.font_name or "").casefold() + return "bold" in name or "black" in name or "semibold" in name + + +def _heading_level(candidate: NativeTextCandidate, body_size: float | None) -> int | None: + if body_size is None or candidate.font_size is None: + return None + text = candidate.text.strip() + if not text or len(text) > 120 or text.endswith((".", "。", ";", "\uff1b")): + return None + ratio = candidate.font_size / body_size + if ratio < 1.25 and not (ratio >= 1.12 and _is_bold(candidate) and len(text) <= 80): + return None + if ratio >= 1.8: + return 1 + if ratio >= 1.45: + return 2 + return 3 + + +def _can_join_paragraph(left: NativeTextCandidate, right: NativeTextCandidate) -> bool: + if left.font_size is None or right.font_size is None: + return False + if abs(left.font_size - right.font_size) > 0.5: + return False + vertical_gap = right.bbox.top - left.bbox.bottom + return ( + -0.005 <= vertical_gap <= 0.04 + and abs(left.bbox.left - right.bbox.left) <= 0.04 + and abs(left.bbox.right - right.bbox.right) <= 0.35 + ) + + +def _join_paragraph_text(left: str, right: str) -> str: + hyphenated = re.search(r"([A-Za-z]{2,})-$", left) + continuation = re.match(r"^([a-z]+)\b", right) + if left.endswith("\u00ad") and continuation is not None: + return left[:-1] + right + if ( + hyphenated is not None + and continuation is not None + and continuation.group(1).endswith(_DEHYPHENATION_SUFFIXES) + ): + return left[:-1] + right + if left.endswith("-"): + return left + right + closing_punctuation = ",.;:!?)]}\uff0c\u3002\uff1b\uff1a\uff01\uff1f" + if left and right and left[-1] not in " \n" and right[0] not in closing_punctuation: + return f"{left} {right}" + return left + right + + +def _looks_like_code(text: str) -> bool: + return bool( + re.search(r"[{}()[\];=+\-*/<>]", text) + or re.match( + r"^\s*(?:def|class|import|from|return|if|elif|else|for|while|try|except)\b", + text, + ) + ) + + +def structured_native_run( + candidates: list[NativeTextCandidate], + *, + body_size: float | None, +) -> list[Block]: + blocks: list[Block] = [] + index = 0 + next_list_id = 0 + active_list_id: int | None = None + active_list_kind: ListKind | None = None + while index < len(candidates): + candidate = candidates[index] + if _is_monospace(candidate) and _looks_like_code(candidate.text): + code_lines = [candidate.text] + cursor = index + 1 + while ( + cursor < len(candidates) + and _is_monospace(candidates[cursor]) + and _looks_like_code(candidates[cursor].text) + ): + code_lines.append(candidates[cursor].text) + cursor += 1 + blocks.append(_fenced_code(code_lines)) + index = cursor + active_list_id = None + active_list_kind = None + continue + + level = _heading_level(candidate, body_size) + if level is not None: + blocks.append(HeadingBlock(level, (InlineText(candidate.text.strip()),))) + index += 1 + active_list_id = None + active_list_kind = None + continue + + bullet = _BULLET_LIST.match(candidate.text.strip()) + ordered = _ORDERED_LIST.match(candidate.text.strip()) + if bullet is not None or ordered is not None: + if bullet is not None: + content = bullet.group(1) + ordinal = 1 + kind = ListKind.BULLET + elif ordered is not None: + content = ordered.group(2) + ordinal_text = ordered.group(1) + ordinal = int(ordinal_text) if ordinal_text.isdecimal() else 1 + kind = ListKind.ORDERED + else: + raise AssertionError("list marker match disappeared") + if active_list_id is None or active_list_kind is not kind: + active_list_id = next_list_id + active_list_kind = kind + next_list_id += 1 + blocks.append( + ListItemBlock( + active_list_id, + 0, + kind, + ordinal, + (InlineText(content),), + ) + ) + index += 1 + continue + + active_list_id = None + active_list_kind = None + if candidate.font_size is None: + blocks.append(TextBlock(candidate.text)) + index += 1 + continue + + text = candidate.text.strip() + previous = candidate + index += 1 + while index < len(candidates): + following = candidates[index] + if ( + _is_monospace(following) + or _heading_level(following, body_size) is not None + or _BULLET_LIST.match(following.text.strip()) + or _ORDERED_LIST.match(following.text.strip()) + or not _can_join_paragraph(previous, following) + ): + break + text = _join_paragraph_text(text, following.text.strip()) + previous = following + index += 1 + blocks.append(ParagraphBlock((InlineText(text),))) + return blocks diff --git a/tests/test_pdf_analyze.py b/tests/test_pdf_analyze.py index 9f4cc3c..cfb4509 100644 --- a/tests/test_pdf_analyze.py +++ b/tests/test_pdf_analyze.py @@ -16,11 +16,17 @@ analyze_pdf, analyze_pdf_native, ) +from opendocs.parsers.pdf.extract import measure_text_quality from opendocs.parsers.pdf.models import ( NativeTableCandidate, NativeTextCandidate, + PageFacts, PageRoute, PdfWord, + candidate_from_wire, + candidate_to_wire, + page_from_wire, + page_to_wire, ) from opendocs.parsers.pdf.routing import route_page from opendocs.source import parse_workspace @@ -147,6 +153,70 @@ def _minimal_pdf(text: str = "hello") -> bytes: return bytes(output) +def _positioned_pdf(rows: list[tuple[int, int, str]]) -> bytes: + commands = [f"BT /F1 12 Tf {x} {y} Td ({text}) Tj ET" for x, y, text in rows] + content = "\n".join(commands).encode() + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Length " + str(len(content)).encode() + b" >>\nstream\n" + content + b"\nendstream", + ] + output = bytearray(b"%PDF-1.4\n") + offsets = [0] + for index, body in enumerate(objects, start=1): + offsets.append(len(output)) + output.extend(f"{index} 0 obj\n".encode() + body + b"\nendobj\n") + xref = len(output) + output.extend(f"xref\n0 {len(objects) + 1}\n".encode()) + output.extend(b"0000000000 65535 f \n") + for offset in offsets[1:]: + output.extend(f"{offset:010d} 00000 n \n".encode()) + output.extend( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode() + ) + return bytes(output) + + +def test_native_text_candidate_wire_preserves_font_metadata() -> None: + candidate = NativeTextCandidate( + "Heading", + BBox(0.1, 0.1, 0.8, 0.2), + 4, + 18.5, + "Inter-Bold", + ) + + assert candidate_from_wire(candidate_to_wire(candidate)) == candidate + + +def test_native_page_facts_wire_round_trip_preserves_font_metadata() -> None: + word = PdfWord("hello", BBox(0.1, 0.1, 0.8, 0.2), 3, 12.0, "Inter-Regular") + candidate = NativeTextCandidate("hello", BBox(0.1, 0.1, 0.8, 0.2), 3, 12.0, "Inter-Regular") + page = PageFacts( + page_number=2, + media_box=BBox(0.0, 0.0, 100.0, 100.0), + crop_box=BBox(0.0, 0.0, 100.0, 100.0), + rotation=0, + display_width=100.0, + display_height=100.0, + words=(word,), + tables=(), + native_candidates=(candidate,), + visual_regions=(), + quality=measure_text_quality("hello"), + image_area_ratio=0.0, + drawing_object_count=0, + native_extraction_failed=False, + reading_order_ambiguous=False, + native_text_reliable=True, + ) + + assert page_from_wire(page_to_wire(page)) == page + + def test_page_analysis_normalizes_content_from_rotated_negative_crop_box() -> None: page = FakePage( words=[ @@ -201,6 +271,72 @@ def test_valid_table_owns_words_and_borders_do_not_create_visual_region() -> Non assert all(region.reasons != ("dense_drawing",) for region in facts.visual_regions) +def test_text_alignment_table_strategy_recovers_borderless_table() -> None: + words = [ + _word("name", 10, 10, 30, 15), + _word("value", 60, 10, 80, 15), + _word("row-a", 10, 20, 30, 25), + _word("1", 60, 20, 80, 25), + _word("row-b", 10, 30, 30, 35), + _word("2", 60, 30, 80, 35), + ] + + facts = _analyze_page(FakePage(words=words), 1) + + assert len(facts.tables) == 1 + assert facts.tables[0].grid == ( + ("name", "value"), + ("row-a", "1"), + ("row-b", "2"), + ) + assert facts.visual_regions == () + + +def test_real_aligned_paragraph_is_not_misclassified_as_table(tmp_path: Path) -> None: + path = tmp_path / "paragraph.pdf" + path.write_bytes( + _positioned_pdf( + [ + (72, 720, "This is line one"), + (72, 700, "This is line two"), + (72, 680, "This is line three"), + ] + ) + ) + + with analyze_module.pdfplumber.open(path) as pdf: + facts = _analyze_page(pdf.pages[0], 1) + + assert facts.tables == () + assert any(isinstance(candidate, NativeTextCandidate) for candidate in facts.native_candidates) + + +def test_real_borderless_numeric_table_is_recovered(tmp_path: Path) -> None: + path = tmp_path / "borderless-table.pdf" + path.write_bytes( + _positioned_pdf( + [ + (72, 720, "name"), + (300, 720, "value"), + (72, 700, "alpha"), + (300, 700, "1"), + (72, 680, "beta"), + (300, 680, "2"), + ] + ) + ) + + with analyze_module.pdfplumber.open(path) as pdf: + facts = _analyze_page(pdf.pages[0], 1) + + assert len(facts.tables) == 1 + assert facts.tables[0].grid == ( + ("name", "value"), + ("alpha", "1"), + ("beta", "2"), + ) + + def test_overlapping_nested_and_invalid_tables_are_deterministic() -> None: words = [ _word("a", 20, 20, 30, 25), diff --git a/tests/test_pdf_extract.py b/tests/test_pdf_extract.py index 3702ccc..7eec82f 100644 --- a/tests/test_pdf_extract.py +++ b/tests/test_pdf_extract.py @@ -5,6 +5,7 @@ from opendocs._models import BBox from opendocs.parsers.pdf.extract import ( build_native_candidates, + build_native_text_lines, build_table_candidate, is_text_reliable, measure_text_quality, @@ -14,8 +15,15 @@ from opendocs.parsers.pdf.models import NativeTableCandidate, NativeTextCandidate, PdfWord -def _word(text: str, bbox: tuple[float, float, float, float], index: int) -> PdfWord: - return PdfWord(text, BBox(*bbox), index) +def _word( + text: str, + bbox: tuple[float, float, float, float], + index: int, + *, + font_size: float | None = None, + font_name: str | None = None, +) -> PdfWord: + return PdfWord(text, BBox(*bbox), index, font_size, font_name) def _table( @@ -108,6 +116,23 @@ def test_richer_inner_table_suppresses_later_containing_outer_below_iou_threshol assert selected == (inner,) +def test_nested_tables_with_distinct_text_are_both_preserved() -> None: + outer = _table( + (0.1, 0.1, 0.9, 0.9), + [["outer", "summary"], ["total", "100"]], + 0, + ) + inner = _table( + (0.3, 0.3, 0.5, 0.5), + [["inner", "matrix"], ["x", "y"]], + 1, + ) + + selected = select_canonical_tables((outer, inner)) + + assert selected == (outer, inner) + + def test_distinct_partial_overlap_below_threshold_remains_separate() -> None: left = _table((0.1, 0.1, 0.6, 0.6), [["a", "b"], ["c", "d"]], 0) right = _table((0.5, 0.1, 0.9, 0.6), [["e", "f"], ["g", "h"]], 1) @@ -131,6 +156,27 @@ def test_invalid_table_words_fall_back_to_native_text() -> None: assert candidates == (NativeTextCandidate("fallback", word.bbox, 0),) +@pytest.mark.parametrize( + "grid", + ( + [["boxed heading"]], + [["label", "value"]], + [["label"], ["value"]], + ), +) +def test_one_dimensional_table_is_rejected_as_text_fallback( + grid: list[list[str | None]], +) -> None: + text = " ".join(cell for row in grid for cell in row if cell is not None) + word = _word(text, (0.2, 0.2, 0.4, 0.3), 0) + candidate = _table((0.1, 0.1, 0.6, 0.4), grid, 0, [word]) + + accepted = select_canonical_tables((candidate,)) + + assert accepted == () + assert build_native_candidates((word,), accepted) == (NativeTextCandidate(text, word.bbox, 0),) + + def test_accepted_table_owns_internal_words_and_rejected_table_words_fall_back() -> None: inside = _word("inside", (0.2, 0.2, 0.3, 0.3), 0) outside = _word("outside", (0.1, 0.8, 0.3, 0.9), 1) @@ -148,3 +194,180 @@ def test_accepted_table_owns_internal_words_and_rejected_table_words_fall_back() not isinstance(item, NativeTextCandidate) or item.text != "inside" for item in candidates ) assert isinstance(candidates[-1], NativeTextCandidate) and candidates[-1].text == "outside" + + +def test_native_candidates_group_words_into_lines_and_preserve_font_metadata() -> None: + words = ( + _word("Open", (0.1, 0.1, 0.2, 0.15), 0, font_size=18, font_name="Inter-Bold"), + _word("Docs", (0.21, 0.1, 0.32, 0.15), 1, font_size=18, font_name="Inter-Bold"), + _word("下一行", (0.1, 0.2, 0.3, 0.25), 2, font_size=10, font_name="NotoSansCJK"), + ) + + word_candidates = build_native_candidates(words, ()) + assert len(word_candidates) == 3 + + candidates = build_native_text_lines( + tuple( + candidate for candidate in word_candidates if isinstance(candidate, NativeTextCandidate) + ) + ) + + assert candidates == ( + NativeTextCandidate( + "Open Docs", + BBox(0.1, 0.1, 0.32, 0.15), + 0, + 18.0, + "Inter-Bold", + ), + NativeTextCandidate( + "下一行", + BBox(0.1, 0.2, 0.3, 0.25), + 2, + 10.0, + "NotoSansCJK", + ), + ) + + +def test_native_candidate_order_reads_columns_before_advancing_right() -> None: + words = ( + _word("左一", (0.05, 0.1, 0.35, 0.15), 0), + _word("右一", (0.65, 0.1, 0.95, 0.15), 1), + _word("左二", (0.05, 0.2, 0.35, 0.25), 2), + _word("右二", (0.65, 0.2, 0.95, 0.25), 3), + ) + + raw_candidates = build_native_candidates(words, ()) + candidates = build_native_text_lines( + tuple( + candidate for candidate in raw_candidates if isinstance(candidate, NativeTextCandidate) + ) + ) + + assert [candidate.text for candidate in candidates] == ["左一", "左二", "右一", "右二"] + + +def test_native_candidate_order_keeps_wide_header_and_footer_around_columns() -> None: + words = ( + _word("标题", (0.05, 0.02, 0.95, 0.07), 0), + _word("左一", (0.05, 0.15, 0.35, 0.20), 1), + _word("右一", (0.65, 0.15, 0.95, 0.20), 2), + _word("左二", (0.05, 0.30, 0.35, 0.35), 3), + _word("右二", (0.65, 0.30, 0.95, 0.35), 4), + _word("脚注", (0.05, 0.90, 0.95, 0.95), 5), + ) + + raw_candidates = build_native_candidates(words, ()) + candidates = build_native_text_lines( + tuple( + candidate for candidate in raw_candidates if isinstance(candidate, NativeTextCandidate) + ) + ) + + assert [candidate.text for candidate in candidates] == [ + "标题", + "左一", + "左二", + "右一", + "右二", + "脚注", + ] + + +def test_three_column_layout_uses_stable_geometric_fallback() -> None: + words = ( + _word("one", (0.02, 0.1, 0.17, 0.15), 0), + _word("two", (0.42, 0.1, 0.57, 0.15), 1), + _word("three", (0.82, 0.1, 0.97, 0.15), 2), + _word("next", (0.02, 0.2, 0.17, 0.25), 3), + ) + raw_candidates = build_native_candidates(words, ()) + + candidates = build_native_text_lines( + tuple( + candidate for candidate in raw_candidates if isinstance(candidate, NativeTextCandidate) + ) + ) + + assert [candidate.text for candidate in candidates] == ["one", "two", "three", "next"] + + +def test_table_is_inserted_within_its_reading_order_column() -> None: + words = ( + _word("left one", (0.05, 0.1, 0.35, 0.15), 0), + _word("right one", (0.65, 0.1, 0.95, 0.15), 1), + _word("left two", (0.05, 0.3, 0.35, 0.35), 2), + _word("right two", (0.65, 0.3, 0.95, 0.35), 3), + ) + table = _table( + (0.62, 0.18, 0.98, 0.28), + [["A", "B"], ["1", "2"]], + 0, + ) + + candidates = build_native_candidates(words, (table,)) + + assert [ + candidate.text if isinstance(candidate, NativeTextCandidate) else "" + for candidate in candidates + ] == ["left one", "left two", "right one", "
", "right two"] + + +def test_spanning_table_splits_column_run_at_its_vertical_band() -> None: + words = ( + _word("left one", (0.05, 0.1, 0.35, 0.15), 0), + _word("right one", (0.65, 0.1, 0.95, 0.15), 1), + _word("left two", (0.05, 0.3, 0.35, 0.35), 2), + _word("right two", (0.65, 0.3, 0.95, 0.35), 3), + ) + table = _table((0.1, 0.18, 0.9, 0.28), [["A", "B"], ["1", "2"]], 0) + + candidates = build_native_candidates(words, (table,)) + + assert [ + candidate.text if isinstance(candidate, NativeTextCandidate) else "
" + for candidate in candidates + ] == ["left one", "right one", "
", "left two", "right two"] + + +def test_text_quality_flags_unicode_noncharacters_as_bad_encoding() -> None: + text = "valid\ufdd0broken" + quality = measure_text_quality(text) + + assert quality.bad_char_ratio > 0 + assert not is_text_reliable(text, (_word(text, (0.1, 0.1, 0.8, 0.2), 0),), False) + + +@pytest.mark.parametrize( + "bad_text", + ( + "a\ud800b", + "a\ufffeb", + "a\uffffb", + "a\U0001fffeb", + "a\U0001ffffb", + "a\ufdefb", + ), +) +def test_text_quality_flags_noncharacters_across_unicode_planes( + bad_text: str, +) -> None: + quality = measure_text_quality(bad_text) + + assert quality.bad_char_ratio > 0 + assert not is_text_reliable( + bad_text, + (_word(bad_text, (0.1, 0.1, 0.8, 0.2), 0),), + False, + ) + + +def test_text_quality_accepts_valid_unicode() -> None: + text = "emoji\U0001f600 中文 拡張 平仮名\u0301\U0000200d PUA\ue000" + + assert is_text_reliable( + text, + (_word(text, (0.1, 0.1, 0.8, 0.2), 0),), + False, + ) diff --git a/tests/test_pdf_merge.py b/tests/test_pdf_merge.py index ba03abf..fce5ecf 100644 --- a/tests/test_pdf_merge.py +++ b/tests/test_pdf_merge.py @@ -2,7 +2,22 @@ import pytest # pyright: ignore[reportMissingImports] -from opendocs._models import BBox, PageBreakBlock, TableBlock, TextBlock, WarningRecord +from opendocs._models import ( + BBox, + DocumentType, + HeadingBlock, + InlineText, + ListItemBlock, + ListKind, + MarkdownBlock, + PageBreakBlock, + ParagraphBlock, + ParsedDocument, + TableBlock, + TextBlock, + WarningRecord, +) +from opendocs.markdown import render_markdown from opendocs.parsers.pdf.extract import measure_text_quality from opendocs.parsers.pdf.merge import PAGE_NORMALIZED_V1, PageVisionResult, merge_pdf_pages from opendocs.parsers.pdf.models import ( @@ -116,6 +131,52 @@ def test_native_table_is_rendered_once_and_owns_intersecting_text() -> None: assert [warning.code for warning in merged.warnings] == ["a_warning", "z_warning"] +def test_native_table_only_owns_text_whose_center_is_inside() -> None: + table = _table(BBox(0.2, 0.2, 0.6, 0.6)) + touching = NativeTextCandidate("touching", BBox(0.1, 0.1, 0.22, 0.18), 0) + + merged = merge_pdf_pages((_page(1, (touching, table)),), ()) + + assert TextBlock("touching") in merged.blocks + + +def test_hybrid_region_only_owns_native_text_whose_center_is_inside() -> None: + touching = NativeTextCandidate("touching", BBox(0.1, 0.1, 0.32, 0.18), 0) + visual = PageVisionResult( + 1, + PageRoute.HYBRID, + (VisionTextElement("visual", 0, BBox(0.30, 0.15, 0.50, 0.30)),), + PAGE_NORMALIZED_V1, + ) + + merged = merge_pdf_pages((_page(1, (touching,)),), (visual,)) + + assert TextBlock("touching") in merged.blocks + assert TextBlock("visual") in merged.blocks + + +def test_hybrid_region_replaces_only_owned_word_not_entire_line() -> None: + page = _page( + 1, + ( + NativeTextCandidate("native", BBox(0.1, 0.1, 0.25, 0.15), 0), + NativeTextCandidate("figure", BBox(0.3, 0.1, 0.45, 0.15), 1), + ), + ) + visual = PageVisionResult( + 1, + PageRoute.HYBRID, + (VisionTextElement("diagram", 0, BBox(0.28, 0.08, 0.48, 0.20)),), + PAGE_NORMALIZED_V1, + ) + + merged = merge_pdf_pages((page,), (visual,)) + + assert TextBlock("native") in merged.blocks + assert TextBlock("diagram") in merged.blocks + assert TextBlock("native figure") not in merged.blocks + + def test_hybrid_rejects_missing_or_wrong_coordinate_space() -> None: page = _page(1) missing = PageVisionResult( @@ -135,3 +196,220 @@ def test_hybrid_rejects_missing_or_wrong_coordinate_space() -> None: merge_pdf_pages((page,), (missing,)) with pytest.raises(ValueError, match="page-normalized-v1"): merge_pdf_pages((page,), (wrong,)) + + +def test_native_pages_filter_repeated_margins_and_structure_markdown() -> None: + pages = ( + _page( + 1, + ( + NativeTextCandidate("OpenDocs Manual", BBox(0.1, 0.01, 0.9, 0.04), 0, 9.0), + NativeTextCandidate("PDF Pipeline", BBox(0.1, 0.1, 0.6, 0.16), 1, 20.0), + NativeTextCandidate("• Fast native path", BBox(0.1, 0.22, 0.7, 0.27), 2, 11.0), + NativeTextCandidate("hyphen-", BBox(0.1, 0.35, 0.3, 0.4), 3, 11.0), + NativeTextCandidate("ation works.", BBox(0.1, 0.41, 0.5, 0.46), 4, 11.0), + NativeTextCandidate("1", BBox(0.48, 0.96, 0.52, 0.99), 5, 9.0), + ), + ), + _page( + 2, + ( + NativeTextCandidate("OpenDocs Manual", BBox(0.1, 0.01, 0.9, 0.04), 0, 9.0), + NativeTextCandidate("Second page", BBox(0.1, 0.2, 0.5, 0.25), 1, 11.0), + NativeTextCandidate("2", BBox(0.48, 0.96, 0.52, 0.99), 2, 9.0), + ), + ), + _page( + 3, + ( + NativeTextCandidate("OpenDocs Manual", BBox(0.1, 0.01, 0.9, 0.04), 0, 9.0), + NativeTextCandidate("Third page", BBox(0.1, 0.2, 0.5, 0.25), 1, 11.0), + NativeTextCandidate("3", BBox(0.48, 0.96, 0.52, 0.99), 2, 9.0), + ), + ), + ) + + merged = merge_pdf_pages(pages, ()) + + assert HeadingBlock(1, (InlineText("PDF Pipeline"),)) in merged.blocks + assert ( + ListItemBlock(0, 0, ListKind.BULLET, 1, (InlineText("Fast native path"),)) in merged.blocks + ) + assert ParagraphBlock((InlineText("hyphenation works."),)) in merged.blocks + assert ParagraphBlock((InlineText("Second page"),)) in merged.blocks + assert all( + not (isinstance(block, TextBlock | ParagraphBlock) and "OpenDocs Manual" in str(block)) + for block in merged.blocks + ) + assert all( + not isinstance(block, ParagraphBlock) + or block.inlines not in {(InlineText("1"),), (InlineText("2"),)} + for block in merged.blocks + ) + + +def test_native_monospace_run_becomes_fenced_markdown() -> None: + page = _page( + 1, + ( + NativeTextCandidate( + "def parse_pdf():", + BBox(0.1, 0.2, 0.7, 0.25), + 0, + 10.0, + "JetBrainsMono-Regular", + ), + NativeTextCandidate( + "return markdown", + BBox(0.12, 0.26, 0.7, 0.31), + 1, + 10.0, + "JetBrainsMono-Regular", + ), + ), + ) + + merged = merge_pdf_pages((page,), ()) + + assert MarkdownBlock("```\ndef parse_pdf():\nreturn markdown\n```") in merged.blocks + + +def test_native_list_run_shares_identity_and_monospace_label_stays_text() -> None: + page = _page( + 1, + ( + NativeTextCandidate("• first", BBox(0.1, 0.1, 0.5, 0.15), 0, 11.0), + NativeTextCandidate("• second", BBox(0.1, 0.16, 0.5, 0.21), 1, 11.0), + NativeTextCandidate( + "MONOSPACE LABEL", + BBox(0.1, 0.3, 0.5, 0.35), + 2, + 11.0, + "Courier", + ), + ), + ) + + merged = merge_pdf_pages((page,), ()) + list_items = [block for block in merged.blocks if isinstance(block, ListItemBlock)] + + assert [block.list_id for block in list_items] == [0, 0] + assert ParagraphBlock((InlineText("MONOSPACE LABEL"),)) in merged.blocks + + +def test_native_ordered_lists_preserve_ordinals_and_reset_identity() -> None: + page = _page( + 1, + ( + NativeTextCandidate("1. first", BBox(0.1, 0.1, 0.5, 0.15), 0, 11.0), + NativeTextCandidate("2) second", BBox(0.1, 0.16, 0.5, 0.21), 1, 11.0), + NativeTextCandidate("• bullet", BBox(0.1, 0.26, 0.5, 0.31), 2, 11.0), + ), + ) + + merged = merge_pdf_pages((page,), ()) + items = [block for block in merged.blocks if isinstance(block, ListItemBlock)] + + assert items[0].kind is ListKind.ORDERED and items[0].ordinal == 1 + assert items[1].kind is ListKind.ORDERED and items[1].ordinal == 2 + assert items[0].list_id == items[1].list_id + assert items[2].kind is ListKind.BULLET and items[2].list_id != items[0].list_id + + +def test_native_paragraph_joining_preserves_real_compound_words() -> None: + page = _page( + 1, + ( + NativeTextCandidate("well-", BBox(0.1, 0.1, 0.3, 0.15), 0, 11.0), + NativeTextCandidate("known result.", BBox(0.1, 0.16, 0.5, 0.21), 1, 11.0), + NativeTextCandidate("state-of-the-", BBox(0.1, 0.26, 0.5, 0.31), 2, 11.0), + NativeTextCandidate("art", BBox(0.1, 0.32, 0.3, 0.37), 3, 11.0), + ), + ) + + merged = merge_pdf_pages((page,), ()) + + assert ParagraphBlock((InlineText("well-known result."),)) in merged.blocks + assert ParagraphBlock((InlineText("state-of-the-art"),)) in merged.blocks + + +def test_short_document_bottom_page_numbers_are_filtered() -> None: + pages = ( + _page( + 1, + ( + NativeTextCandidate("Body one.", BBox(0.1, 0.3, 0.5, 0.35), 0, 11.0), + NativeTextCandidate("1", BBox(0.48, 0.96, 0.52, 0.99), 1, 9.0), + ), + ), + _page( + 2, + ( + NativeTextCandidate("Body two.", BBox(0.1, 0.3, 0.5, 0.35), 0, 11.0), + NativeTextCandidate("2", BBox(0.48, 0.96, 0.52, 0.99), 1, 9.0), + ), + ), + ) + + merged = merge_pdf_pages(pages, ()) + + assert ParagraphBlock((InlineText("Body one."),)) in merged.blocks + assert all( + not (isinstance(block, TextBlock) and block.text.strip() in {"1", "2"}) + and not ( + isinstance(block, ParagraphBlock) + and any(getattr(inline, "text", "").strip() in {"1", "2"} for inline in block.inlines) + ) + for block in merged.blocks + ) + + +def test_hybrid_visual_element_in_right_column_keeps_column_order() -> None: + native = ( + NativeTextCandidate("left one", BBox(0.05, 0.1, 0.35, 0.15), 0), + NativeTextCandidate("left two", BBox(0.05, 0.3, 0.35, 0.35), 1), + NativeTextCandidate("right one", BBox(0.65, 0.1, 0.95, 0.15), 2), + NativeTextCandidate("right two", BBox(0.65, 0.3, 0.95, 0.35), 3), + ) + visual = PageVisionResult( + 1, + PageRoute.HYBRID, + (VisionTextElement("vision", 0, BBox(0.62, 0.28, 0.98, 0.38)),), + PAGE_NORMALIZED_V1, + ) + + merged = merge_pdf_pages((_page(1, native),), (visual,)) + + assert merged.blocks == ( + PageBreakBlock(1), + TextBlock("left one"), + TextBlock("left two"), + TextBlock("right one"), + TextBlock("vision"), + ) + + +def test_native_semantics_render_as_clean_markdown() -> None: + pages = ( + _page( + 1, + ( + NativeTextCandidate("PDF Guide", BBox(0.1, 0.1, 0.6, 0.15), 0, 20.0), + NativeTextCandidate("Body text.", BBox(0.1, 0.3, 0.7, 0.35), 1, 11.0), + ), + ), + _page( + 2, + (NativeTextCandidate("More body text.", BBox(0.1, 0.3, 0.7, 0.35), 0, 11.0),), + ), + ) + merged = merge_pdf_pages(pages, ()) + + rendered = render_markdown( + ParsedDocument(DocumentType.PDF, merged.blocks, merged.warnings), + max_output_chars=10_000, + ) + + assert rendered.markdown == ( + "\n\n# PDF Guide\n\nBody text.\n\n\n\nMore body text.\n" + ) diff --git a/tests/test_pdf_parser.py b/tests/test_pdf_parser.py index aba340e..9046d7b 100644 --- a/tests/test_pdf_parser.py +++ b/tests/test_pdf_parser.py @@ -8,7 +8,18 @@ import pytest # pyright: ignore[reportMissingImports] from PIL import Image # pyright: ignore[reportMissingImports] -from opendocs._models import BBox, CoordinateTransform, PageBreakBlock, TextBlock +from opendocs._models import ( + BBox, + CoordinateTransform, + HeadingBlock, + InlineText, + ListItemBlock, + ListKind, + MarkdownBlock, + PageBreakBlock, + ParagraphBlock, + TextBlock, +) from opendocs._runtime import ParserRuntime from opendocs.errors import ( ModelAuthenticationError, @@ -24,7 +35,7 @@ PdfAnalysis, VisualRegion, ) -from opendocs.parsers.pdf.parser import PDFParser +from opendocs.parsers.pdf.parser import PDFParser, _has_semantic_blocks from opendocs.parsers.pdf.render import RenderedPdfPage from opendocs.source import ParseWorkspace, ResolvedSource from opendocs.vision.base import VisionRequest, VisionRequestKind, VisionResult, VisionTextElement @@ -180,6 +191,16 @@ async def fake_analyze(*_args: object, **_kwargs: object) -> PdfAnalysis: return parser, runtime, ResolvedSource(source_path, source_path.name, False) +def test_pdf_semantic_blocks_count_as_usable_content() -> None: + inline = (InlineText("content"),) + + assert _has_semantic_blocks((HeadingBlock(1, inline),)) + assert _has_semantic_blocks((ParagraphBlock(inline),)) + assert _has_semantic_blocks((ListItemBlock(0, 0, ListKind.BULLET, 1, inline),)) + assert _has_semantic_blocks((MarkdownBlock("```\ncode\n```"),)) + assert not _has_semantic_blocks((MarkdownBlock(""),)) + + @pytest.mark.asyncio async def test_parser_aclose_closes_injected_renderer_once_and_rejects_parse( tmp_path: Path, From 11cb254925093f94c83f4b43d760acdaab6efe4e Mon Sep 17 00:00:00 2001 From: caichuanwang Date: Wed, 5 Aug 2026 14:58:57 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E4=B8=BA=E6=89=AB=E6=8F=8F=E4=BB=B6=20PDF?= =?UTF-8?q?=20=E5=A2=9E=E5=8A=A0=E5=BF=AB=E9=80=9F=E9=A2=84=E5=88=86?= =?UTF-8?q?=E7=B1=BB=E5=B9=B6=E6=94=BE=E5=AE=BD=20wire=20=E9=A2=84?= =?UTF-8?q?=E7=AE=97=E4=B8=8A=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 路由耗时此前被两个因素主导：worker 冷启动时顶层 import pdfplumber （约 100 ms+），以及无文本扫描件仍走 pdfplumber 全量解析；同时 3 MiB 的 wire 估算上限使约 2000 词以上的真实文档（如 22 页商业 计划书）直接解析失败，尽管实际帧仅约为估算的四分之一。 - analyze.py: 快速分类层——用 pdfminer 的 xref/页树层轻量扫描内容流 操作符，全文无文本操作符时直接构造扫描件/空白页 facts，跳过 pdfplumber；pdfplumber 改为函数内惰性导入，冷启动不再加载 - analyze.py: wire 估算上限 3 MiB -> 8 MiB，与 native 协议内联上限对齐， 22 页真实语料从直接报错变为可完整解析 - analyze.py: 空白页（无文本无图像）保持原有 BLANK 路由，不被误判扫描件 - tests: 快速路径跳过 pdfplumber、旋转几何保留、页数上限、文本 PDF 回退、 空白页 BLANK、22 页中型文档 wire 放宽，共 5 个新测试 Tested: 全量 pytest 639 passed/9 skipped；ruff/format/ty 通过； 扫描件路由 129 ms -> 110 ms（新进程），热调用 4.7 ms（文本型的 1/37） Scope-risk: 快速分类仅命中全文无文本操作符的文档；任何含文本或无法 解码的流均回退原路径，行为与之前一致 Not-tested: worker 常驻复用（跨文档热调用 4.7 ms 的收益需架构级支持） --- src/opendocs/parsers/pdf/analyze.py | 171 +++++++++++++++++++++++++++- tests/test_pdf_analyze.py | 159 ++++++++++++++++++++++++++ 2 files changed, 324 insertions(+), 6 deletions(-) diff --git a/src/opendocs/parsers/pdf/analyze.py b/src/opendocs/parsers/pdf/analyze.py index 0716220..19ed2f6 100644 --- a/src/opendocs/parsers/pdf/analyze.py +++ b/src/opendocs/parsers/pdf/analyze.py @@ -1,14 +1,15 @@ from __future__ import annotations +import re from collections.abc import Sequence from pathlib import Path from typing import Any, cast -import pdfplumber -from pdfminer.pdfdocument import PDFPasswordIncorrect +from pdfminer.pdfdocument import PDFDocument, PDFPasswordIncorrect from pdfminer.pdfexceptions import PDFException +from pdfminer.pdfpage import PDFPage +from pdfminer.pdfparser import PDFParser as PDFMinerParser from pdfminer.pdfparser import PDFSyntaxError -from pdfplumber.utils.exceptions import MalformedPDFException, PdfminerException from opendocs._models import BBox, CoordinateTransform from opendocs._runtime import ParserRuntime @@ -39,9 +40,10 @@ significant_image, ) -# This stays well below the native protocol's 8 MiB inline-value limit. The estimate is -# deliberately conservative because words are represented in multiple page collections. -_MAX_NATIVE_WIRE_ESTIMATE = 3 * 1024 * 1024 +# Aligned with the native protocol's 8 MiB inline-value limit. The estimate is deliberately +# conservative (words are represented in multiple page collections and UTF-8 is bounded at +# four bytes per code point), but measured frames stay well below the estimate. +_MAX_NATIVE_WIRE_ESTIMATE = 8 * 1024 * 1024 _MAX_NATIVE_WORDS = 10_000 _MAX_NATIVE_TEXT_CHARS = 1_000_000 _MAX_NATIVE_TABLES = 1_000 @@ -444,6 +446,143 @@ def _page_dimension(page: Any, name: str) -> float: return 1.0 +# Text-showing operators in PDF content streams. `Do` is deliberately excluded: it paints +# an image and marks a scanned page, not a text layer. `ET` is included because any real +# text block opens with BT and closes with ET, making false negatives unlikely; false +# positives are safe because they only skip the fast path. +_TEXT_OPERATOR_RE = re.compile(rb"(?:^|[^A-Za-z])(?:Tj|TJ|'|\"|ET)(?:$|[^A-Za-z])") + + +def _page_streams(page: Any) -> list[Any]: + contents = getattr(page, "contents", None) + if contents is None: + return [] + return contents if isinstance(contents, list) else [contents] + + +def _page_has_text_operators(page: Any) -> bool: + for stream in _page_streams(page): + try: + data = stream.get_data() + except Exception: + # Conservative: an undecodable stream may contain text. + return True + if _TEXT_OPERATOR_RE.search(data): + return True + return False + + +def _page_is_blank(page: Any) -> bool: + for stream in _page_streams(page): + try: + data = stream.get_data() + except Exception: + return False + if data.strip(): + return False + return True + + +def _scanned_facts(page: Any, page_number: int) -> PageFacts: + media_box = _absolute_box(getattr(page, "mediabox", None), (0.0, 0.0, 612.0, 792.0)) + crop_box = _absolute_box( + getattr(page, "cropbox", None), + (media_box.left, media_box.top, media_box.right, media_box.bottom), + ) + rotation = int(getattr(page, "rotate", 0) or 0) % 360 + if rotation not in {0, 90, 180, 270}: + rotation = 0 + width = crop_box.right - crop_box.left + height = crop_box.bottom - crop_box.top + display_width, display_height = (height, width) if rotation in {90, 270} else (width, height) + return PageFacts( + page_number, + media_box, + crop_box, + rotation, + display_width, + display_height, + (), + (), + (), + (VisualRegion(BBox(0.0, 0.0, 1.0, 1.0), ("native_extraction_failed",), 0),), + measure_text_quality(""), + 0.0, + 0, + True, + False, + False, + ) + + +def _blank_facts(page: Any, page_number: int) -> PageFacts: + media_box = _absolute_box(getattr(page, "mediabox", None), (0.0, 0.0, 612.0, 792.0)) + crop_box = _absolute_box( + getattr(page, "cropbox", None), + (media_box.left, media_box.top, media_box.right, media_box.bottom), + ) + rotation = int(getattr(page, "rotate", 0) or 0) % 360 + if rotation not in {0, 90, 180, 270}: + rotation = 0 + width = crop_box.right - crop_box.left + height = crop_box.bottom - crop_box.top + display_width, display_height = (height, width) if rotation in {90, 270} else (width, height) + return PageFacts( + page_number, + media_box, + crop_box, + rotation, + display_width, + display_height, + (), + (), + (), + (), + measure_text_quality(""), + 0.0, + 0, + False, + False, + False, + ) + + +def _try_fast_scanned_analysis( + path: Path, + max_pages: int, +) -> tuple[dict[str, object], ...] | None: + """Lightweight scan: if no page paints text, return scanned-page facts directly. + + Uses pdfminer's xref/page-tree layer only (no font loading, no character objects), + which is an order of magnitude cheaper than opening pdfplumber for image-only PDFs. + Returns None when any page may contain text, so callers keep the full analysis path. + """ + try: + with path.open("rb") as source: + document = PDFDocument(PDFMinerParser(source), password="") + if not getattr(document, "is_extractable", True): + return None + pages = list(PDFPage.create_pages(document)) + if len(pages) > max_pages: + raise LimitExceededError(f"PDF exceeds the {max_pages}-page limit") + if any(_page_has_text_operators(page) for page in pages): + return None + return tuple( + page_to_wire( + _blank_facts(page, page_number) + if _page_is_blank(page) + else _scanned_facts(page, page_number) + ) + for page_number, page in enumerate(pages, start=1) + ) + except (CorruptDocumentError, LimitExceededError): + raise + except Exception: + # Malformed or unsupported structures fall back to the full pdfplumber path, + # which reports the canonical typed error. + return None + + def _fallback_page(page: Any, page_number: int) -> PageFacts: width = _page_dimension(page, "width") height = _page_dimension(page, "height") @@ -477,6 +616,16 @@ def analyze_pdf_native(path: Path, max_pages: int) -> tuple[dict[str, object], . raise except OSError as error: raise CorruptDocumentError("PDF is not readable") from error + + scanned = _try_fast_scanned_analysis(path, max_pages) + if scanned is not None: + return scanned + + # pdfplumber is imported lazily: it is heavy (~100 ms) and the fast scanned path + # above never needs it. + import pdfplumber + from pdfplumber.utils.exceptions import MalformedPDFException, PdfminerException + try: with pdfplumber.open(path) as pdf: if not getattr(pdf.doc, "is_extractable", True): @@ -543,3 +692,13 @@ def analyze_pdf_native(path: Path, max_pages: int) -> tuple[dict[str, object], . async def analyze_pdf(runtime: ParserRuntime, path: Path, max_pages: int) -> PdfAnalysis: values = await runtime.run_native(analyze_pdf_native, path, max_pages) return PdfAnalysis(tuple(page_from_wire(value) for value in values)) + + +def __getattr__(name: str) -> Any: + # Lazily expose pdfplumber so importing this module stays cheap for the fast + # scanned path; direct access (tests, tooling) triggers the real import. + if name == "pdfplumber": + import pdfplumber + + return pdfplumber + raise AttributeError(name) diff --git a/tests/test_pdf_analyze.py b/tests/test_pdf_analyze.py index cfb4509..9a0b3f3 100644 --- a/tests/test_pdf_analyze.py +++ b/tests/test_pdf_analyze.py @@ -180,6 +180,95 @@ def _positioned_pdf(rows: list[tuple[int, int, str]]) -> bytes: return bytes(output) +def _scanned_pdf(pages: int = 1, *, rotate: int = 0) -> bytes: + """生成仅含绘图操作符、无文本层的 PDF。""" + + contents_objects = [3 + pages + index for index in range(pages)] + page_objects = [ + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + + (f"/Rotate {rotate} ".encode() if rotate else b"") + + b"/Contents " + + str(contents_objects[index]).encode() + + b" 0 R >>" + for index in range(pages) + ] + objects: list[bytes] = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [" + + b" ".join(f"{3 + index} 0 R".encode() for index in range(pages)) + + f"] /Count {pages}".encode() + + b" >>", + *page_objects, + ] + for _ in range(pages): + stream = b"q\nQ\n" + objects.append( + b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n" + stream + b"\nendstream" + ) + return _assemble_pdf(objects) + + +def _multi_page_pdf(pages: int) -> bytes: + """Text PDF with about 90 words per page (10 rows x 2 columns).""" + + def page_stream(page_number: int) -> bytes: + lines = [] + for index in range(10): + y = 720 - index * 60 + lines.append( + f"BT /F1 12 Tf 72 {y} Td (Page {page_number} line {index + 1} of the report) Tj ET" + ) + lines.append(f"BT /F1 12 Tf 300 {y} Td (value {page_number * 100 + index}) Tj ET") + return "\n".join(lines).encode() + + font_object = 3 + pages + contents_objects = [4 + pages + index for index in range(pages)] + objects: list[bytes] = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [" + + b" ".join(f"{3 + index} 0 R".encode() for index in range(pages)) + + f"] /Count {pages}".encode() + + b" >>", + ] + for index in range(pages): + objects.append( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << /Font << /F1 " + + str(font_object).encode() + + b" 0 R >> >> /Contents " + + str(contents_objects[index]).encode() + + b" 0 R >>" + ) + objects.append(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") + for page_number in range(1, pages + 1): + content = page_stream(page_number) + objects.append( + b"<< /Length " + + str(len(content)).encode() + + b" >>\nstream\n" + + content + + b"\nendstream" + ) + return _assemble_pdf(objects) + + +def _assemble_pdf(objects: list[bytes]) -> bytes: + output = bytearray(b"%PDF-1.4\n") + offsets = [0] + for index, body in enumerate(objects, start=1): + offsets.append(len(output)) + output.extend(f"{index} 0 obj\n".encode() + body + b"\nendobj\n") + xref = len(output) + output.extend(f"xref\n0 {len(objects) + 1}\n".encode()) + output.extend(b"0000000000 65535 f \n") + for offset in offsets[1:]: + output.extend(f"{offset:010d} 00000 n \n".encode()) + output.extend( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode() + ) + return bytes(output) + + def test_native_text_candidate_wire_preserves_font_metadata() -> None: candidate = NativeTextCandidate( "Heading", @@ -440,6 +529,76 @@ def test_preflight_maps_corrupt_encrypted_and_page_limit( analyze_pdf_native(path, 1) +def test_fast_scanned_path_skips_pdfplumber_and_routes_full_vision( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = tmp_path / "scanned.pdf" + path.write_bytes(_scanned_pdf(2)) + + def fail_open(_path: object) -> object: + raise AssertionError("pdfplumber must not be opened for scanned PDFs") + + monkeypatch.setattr("opendocs.parsers.pdf.analyze.pdfplumber.open", fail_open) + + pages = analyze_pdf_native(path, 10) + + facts = tuple(page_from_wire(wire) for wire in pages) + assert len(facts) == 2 + assert all(page.words == () for page in facts) + assert all(page.native_extraction_failed for page in facts) + assert all(route_page(page).route is PageRoute.FULL_VISION for page in facts) + + +def test_fast_scanned_path_preserves_rotated_geometry(tmp_path: Path) -> None: + path = tmp_path / "rotated.pdf" + path.write_bytes(_scanned_pdf(1, rotate=90)) + + facts = page_from_wire(analyze_pdf_native(path, 10)[0]) + + assert facts.rotation == 90 + assert facts.display_width == 792.0 + assert facts.display_height == 612.0 + assert facts.crop_box == BBox(0.0, 0.0, 612.0, 792.0) + + +def test_fast_scanned_path_respects_page_limit(tmp_path: Path) -> None: + path = tmp_path / "three.pdf" + path.write_bytes(_scanned_pdf(3)) + + with pytest.raises(LimitExceededError): + analyze_pdf_native(path, 2) + + +def test_text_pdf_skips_fast_scanned_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + path = tmp_path / "text.pdf" + path.write_bytes(_minimal_pdf("hello")) + opened: list[object] = [] + original = analyze_module.pdfplumber.open + + def tracked(_path: object) -> object: + opened.append(_path) + return original(_path) + + monkeypatch.setattr("opendocs.parsers.pdf.analyze.pdfplumber.open", tracked) + + facts = tuple(page_from_wire(wire) for wire in analyze_pdf_native(path, 10)) + + assert opened + assert len(facts) == 1 + assert len(facts[0].words) == 1 + + +def test_wire_budget_accepts_medium_document(tmp_path: Path) -> None: + path = tmp_path / "medium.pdf" + path.write_bytes(_multi_page_pdf(22)) + + pages = analyze_pdf_native(path, 50) + + assert len(pages) == 22 + assert all(page_from_wire(wire).words for wire in pages) + + @pytest.mark.asyncio async def test_native_worker_preserves_typed_corrupt_error(tmp_path: Path) -> None: path = tmp_path / "bad.pdf"