diff --git a/docs/JSON_PROTOCOL.md b/docs/JSON_PROTOCOL.md index 3cd4341..1b88206 100644 --- a/docs/JSON_PROTOCOL.md +++ b/docs/JSON_PROTOCOL.md @@ -110,6 +110,20 @@ Structured Content preserves the optional `anchor` in its `captions` and `footnotes` entries; their `content` remains plain rendered annotation content without an injected target tag. +## Cross-page table detection diagnostics + +`merge_table()` accepts an optional `on_decision` callback for callers that need to +explain why a page pair was not marked as a continuation. Each immutable +`TableMergeDecision` contains the two page indices, available table block indices, +and a `TableMergeRejection` reason. `decision.mergeable` is true only when the +existing continuation checks accept the pair. Boundary rejections also identify the +blocking page and block type. + +The callback reports detection decisions in the existing reverse page order. It does +not report whether a later renderer can merge the table content, and it does not +include table text. Omitting the callback preserves the existing behavior and output; +no document schema or serialized diagnostic fields are added. + ## Optional PDF layout geometry Native PDF analysis adds `extensions.docvortex_layout` and deterministic diff --git a/src/docvortex/content/table/__init__.py b/src/docvortex/content/table/__init__.py index f7f21e0..f796f20 100644 --- a/src/docvortex/content/table/__init__.py +++ b/src/docvortex/content/table/__init__.py @@ -3,12 +3,14 @@ from .content import merge_table_content from .document import merge_table from .html import build_row_rendered_cell_segments, build_table_state_from_html, calculate_row_rendered_segments -from .models import RenderedCellSegment +from .models import RenderedCellSegment, TableMergeDecision, TableMergeRejection from .structure import can_merge_by_structure, detect_table_headers from .structure import _expand_header_count_by_rowspan as expand_header_count_by_rowspan __all__ = [ "RenderedCellSegment", + "TableMergeDecision", + "TableMergeRejection", "merge_table", "merge_table_content", "build_table_state_from_html", diff --git a/src/docvortex/content/table/document.py b/src/docvortex/content/table/document.py index 2f4b503..411e290 100644 --- a/src/docvortex/content/table/document.py +++ b/src/docvortex/content/table/document.py @@ -2,13 +2,13 @@ from __future__ import annotations -from typing import Any +from typing import Any, Callable from ...schema import MERGE_TRANSPARENT_BLOCK_TYPES, BlockType from .blocks import _get_or_create_table_state -from .models import BlockDict, PageInfoDict, TableMergeState -from .structure import can_merge_tables +from .models import BlockDict, PageInfoDict, TableMergeDecision, TableMergeRejection, TableMergeState +from .structure import _table_merge_rejection_reason TABLE_BOUNDARY_IGNORED_TYPES = set(MERGE_TRANSPARENT_BLOCK_TYPES) @@ -36,19 +36,29 @@ def _clear_nested_continuation_markers(block: BlockDict) -> None: _clear_nested_continuation_markers(child) -def _find_boundary_table(blocks: list[Any], *, from_end: bool) -> BlockDict | None: - """从页边界扫描 table;噪声块可跳过,其他语义块立即阻断。""" +def _find_boundary_table_with_blocker( + blocks: list[Any], + *, + from_end: bool, +) -> tuple[BlockDict | None, str | None]: + """返回边界表格,或首个阻断块的类型。""" ordered_blocks = reversed(blocks) if from_end else iter(blocks) for block in ordered_blocks: if not isinstance(block, dict): - return None + return None, type(block).__name__ block_type = block.get("type") if block_type in TABLE_BOUNDARY_IGNORED_TYPES: continue if block_type == BlockType.TABLE: - return block - return None - return None + return block, None + return None, block_type if isinstance(block_type, str) else type(block_type).__name__ + return None, None + + +def _find_boundary_table(blocks: list[Any], *, from_end: bool) -> BlockDict | None: + """从页边界扫描 table;噪声块可跳过,其他语义块立即阻断。""" + table_block, _ = _find_boundary_table_with_blocker(blocks, from_end=from_end) + return table_block def _is_consecutive_page_pair(previous_page: PageInfoDict, current_page: PageInfoDict) -> bool: @@ -58,8 +68,61 @@ def _is_consecutive_page_pair(previous_page: PageInfoDict, current_page: PageInf return type(previous_page_idx) is int and type(current_page_idx) is int and current_page_idx == previous_page_idx + 1 -def merge_table(page_info_list: list[PageInfoDict]) -> None: - """倒序识别连续页边界表格,并只在后表写入延续标记。""" +def _page_index(page: Any) -> int | None: + """读取诊断使用的合法页索引,非法页不抛出额外异常。""" + if not isinstance(page, dict): + return None + page_idx = page.get("page_idx") + return page_idx if type(page_idx) is int else None + + +def _table_index(table_block: BlockDict | None) -> int | None: + """读取诊断使用的表格块索引。""" + if not isinstance(table_block, dict): + return None + index = table_block.get("index") + return index if type(index) is int else None + + +def _report_decision( + on_decision: Callable[[TableMergeDecision], None] | None, + *, + previous_page: Any, + current_page: Any, + rejection: TableMergeRejection | None, + previous_table: BlockDict | None = None, + current_table: BlockDict | None = None, + blocker_page: Any = None, + blocker_block_type: str | None = None, +) -> None: + """仅在调用方请求时构造并发送不可变诊断。""" + if on_decision is None: + return + on_decision( + TableMergeDecision( + previous_page_idx=_page_index(previous_page), + current_page_idx=_page_index(current_page), + previous_table_index=_table_index(previous_table), + current_table_index=_table_index(current_table), + rejection=rejection, + blocker_page_idx=_page_index(blocker_page), + blocker_block_type=blocker_block_type, + ) + ) + + +def merge_table( + page_info_list: list[PageInfoDict], + *, + on_decision: Callable[[TableMergeDecision], None] | None = None, +) -> None: + """倒序识别连续页边界表格,并只在后表写入延续标记。 + + ``on_decision`` 可选接收不含正文的检测诊断,不改变默认标记行为。 + 诊断只覆盖延续检测,不承诺后续渲染能够合并表格内容。 + """ + if on_decision is not None and not callable(on_decision): + raise TypeError("on_decision must be callable or None") if not isinstance(page_info_list, list): return @@ -79,28 +142,95 @@ def merge_table(page_info_list: list[PageInfoDict]) -> None: current_page = page_info_list[page_position] previous_page = page_info_list[page_position - 1] if not isinstance(current_page, dict) or not isinstance(previous_page, dict): + _report_decision( + on_decision, + previous_page=previous_page, + current_page=current_page, + rejection=TableMergeRejection.INVALID_PAGE, + ) continue if not _is_consecutive_page_pair(previous_page, current_page): + _report_decision( + on_decision, + previous_page=previous_page, + current_page=current_page, + rejection=TableMergeRejection.NON_CONSECUTIVE_PAGES, + ) continue current_blocks = current_page.get("blocks") previous_blocks = previous_page.get("blocks") if not isinstance(current_blocks, list) or not isinstance(previous_blocks, list): + _report_decision( + on_decision, + previous_page=previous_page, + current_page=current_page, + rejection=TableMergeRejection.INVALID_PAGE, + ) continue - current_table_block = _find_boundary_table(current_blocks, from_end=False) - previous_table_block = _find_boundary_table(previous_blocks, from_end=True) + current_table_block, current_blocker_type = _find_boundary_table_with_blocker(current_blocks, from_end=False) + previous_table_block, previous_blocker_type = _find_boundary_table_with_blocker(previous_blocks, from_end=True) if current_table_block is None or previous_table_block is None: + current_is_blocked = current_table_block is None and current_blocker_type is not None + previous_is_blocked = previous_table_block is None and previous_blocker_type is not None + _report_decision( + on_decision, + previous_page=previous_page, + current_page=current_page, + rejection=( + TableMergeRejection.BOUNDARY_BLOCKED + if current_is_blocked or previous_is_blocked + else TableMergeRejection.BOUNDARY_TABLE_MISSING + ), + previous_table=previous_table_block, + current_table=current_table_block, + blocker_page=current_page if current_is_blocked else previous_page if previous_is_blocked else None, + blocker_block_type=current_blocker_type if current_is_blocked else previous_blocker_type, + ) continue if "_pdf_structure" in current_table_block or "_pdf_structure" in previous_table_block: + _report_decision( + on_decision, + previous_page=previous_page, + current_page=current_page, + rejection=TableMergeRejection.PDF_STRUCTURE_PRESERVED, + previous_table=previous_table_block, + current_table=current_table_block, + ) continue current_state = _get_or_create_table_state(current_table_block, state_cache) previous_state = _get_or_create_table_state(previous_table_block, state_cache) if current_state is None or previous_state is None: + _report_decision( + on_decision, + previous_page=previous_page, + current_page=current_page, + rejection=TableMergeRejection.INVALID_TABLE_STRUCTURE, + previous_table=previous_table_block, + current_table=current_table_block, + ) continue - if not can_merge_tables(current_state, previous_state): + rejection = _table_merge_rejection_reason(current_state, previous_state) + if rejection is not None: + _report_decision( + on_decision, + previous_page=previous_page, + current_page=current_page, + rejection=rejection, + previous_table=previous_table_block, + current_table=current_table_block, + ) continue current_table_block["continues_prev"] = True + _report_decision( + on_decision, + previous_page=previous_page, + current_page=current_page, + rejection=None, + previous_table=previous_table_block, + current_table=current_table_block, + ) diff --git a/src/docvortex/content/table/models.py b/src/docvortex/content/table/models.py index 48a2e68..19ac6f1 100644 --- a/src/docvortex/content/table/models.py +++ b/src/docvortex/content/table/models.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass +from enum import Enum from typing import Any, TypeAlias MAX_HEADER_ROWS = 5 @@ -12,6 +13,40 @@ CalculationBBox: TypeAlias = tuple[int, int, int, int] +class TableMergeRejection(str, Enum): + """跨页表格延续检测的拒绝原因。""" + + INVALID_PAGE = "invalid_page" + NON_CONSECUTIVE_PAGES = "non_consecutive_pages" + BOUNDARY_TABLE_MISSING = "boundary_table_missing" + BOUNDARY_BLOCKED = "boundary_blocked" + PDF_STRUCTURE_PRESERVED = "pdf_structure_preserved" + INVALID_TABLE_STRUCTURE = "invalid_table_structure" + CONTINUATION_CAPTION_REQUIRED = "continuation_caption_required" + MULTIPLE_PREVIOUS_FOOTNOTES = "multiple_previous_footnotes" + PREVIOUS_FOOTNOTES_REQUIRE_CAPTION = "previous_footnotes_require_caption" + INCOMPATIBLE_TABLE_WIDTHS = "incompatible_table_widths" + INCOMPATIBLE_TABLE_STRUCTURE = "incompatible_table_structure" + + +@dataclass(frozen=True, slots=True) +class TableMergeDecision: + """记录一次相邻页表格检测的结果,不包含论文正文。""" + + previous_page_idx: int | None + current_page_idx: int | None + previous_table_index: int | None + current_table_index: int | None + rejection: TableMergeRejection | None + blocker_page_idx: int | None = None + blocker_block_type: str | None = None + + @property + def mergeable(self) -> bool: + """返回相邻页表格是否通过延续检测。""" + return self.rejection is None + + @dataclass class RowMetrics: """记录单行的有效列、实际列和视觉列指标。""" diff --git a/src/docvortex/content/table/structure.py b/src/docvortex/content/table/structure.py index 38766ed..1d53b7a 100644 --- a/src/docvortex/content/table/structure.py +++ b/src/docvortex/content/table/structure.py @@ -10,7 +10,7 @@ from .blocks import _bbox_for_calculation, _is_continuation_caption, _is_post_table_non_continuation_caption, _table_children from .html import _colspan, _rowspan, calculate_row_rendered_segments -from .models import MAX_HEADER_ROWS, TableMergeState +from .models import MAX_HEADER_ROWS, TableMergeRejection, TableMergeState def detect_table_headers( @@ -153,13 +153,16 @@ def _table_widths_are_compatible(current_bbox: Any, previous_bbox: Any) -> bool: return min_width > 0 and abs(current_width - previous_width) / min_width < 0.1 -def can_merge_tables(current_state: TableMergeState, previous_state: TableMergeState) -> bool: - """根据 dict 表格的辅助文本、宽度和 HTML 结构判断是否可合并。""" +def _table_merge_rejection_reason( + current_state: TableMergeState, + previous_state: TableMergeState, +) -> TableMergeRejection | None: + """按原判定顺序返回首个拒绝原因,成功时返回空。""" current_table_block = current_state.owner_block previous_table_block = previous_state.owner_block if not isinstance(previous_table_block, dict) or not isinstance(current_table_block, dict): - return False + return TableMergeRejection.INVALID_TABLE_STRUCTURE previous_children = _table_children(previous_table_block) current_children = _table_children(current_table_block) @@ -172,17 +175,24 @@ def can_merge_tables(current_state: TableMergeState, previous_state: TableMergeS has_continuation_marker = any(_is_continuation_caption(block) for block in merge_caption_blocks) if not has_continuation_marker: - return False + return TableMergeRejection.CONTINUATION_CAPTION_REQUIRED if footnote_count > 1: - return False + return TableMergeRejection.MULTIPLE_PREVIOUS_FOOTNOTES elif footnote_count > 0: - return False + return TableMergeRejection.PREVIOUS_FOOTNOTES_REQUIRE_CAPTION if not _table_widths_are_compatible(current_table_block.get("bbox"), previous_table_block.get("bbox")): - return False + return TableMergeRejection.INCOMPATIBLE_TABLE_WIDTHS + + if not can_merge_by_structure(current_state, previous_state): + return TableMergeRejection.INCOMPATIBLE_TABLE_STRUCTURE + return None - return can_merge_by_structure(current_state, previous_state) + +def can_merge_tables(current_state: TableMergeState, previous_state: TableMergeState) -> bool: + """根据 dict 表格的辅助文本、宽度和 HTML 结构判断是否可合并。""" + return _table_merge_rejection_reason(current_state, previous_state) is None def check_rows_match(previous_state: TableMergeState, current_state: TableMergeState) -> bool: diff --git a/src/docvortex/public_api.py b/src/docvortex/public_api.py index d229306..f3910d9 100644 --- a/src/docvortex/public_api.py +++ b/src/docvortex/public_api.py @@ -133,6 +133,8 @@ "docvortex.content.table": ( "build_row_rendered_cell_segments", "RenderedCellSegment", + "TableMergeDecision", + "TableMergeRejection", "build_table_state_from_html", "calculate_row_rendered_segments", "can_merge_by_structure", diff --git a/tests/unittest/test_table_merge.py b/tests/unittest/test_table_merge.py index 293bffa..c09a58c 100644 --- a/tests/unittest/test_table_merge.py +++ b/tests/unittest/test_table_merge.py @@ -6,7 +6,7 @@ import pytest from bs4 import BeautifulSoup -from docvortex.content.table import merge_table, merge_table_content +from docvortex.content.table import TableMergeDecision, TableMergeRejection, merge_table, merge_table_content from docvortex.schema import BlockType DEFAULT_HTML = "
AB
" @@ -443,3 +443,143 @@ def test_merge_table_content_returns_none_for_invalid_input( assert merge_table_content(previous, current) is None assert previous == previous_original assert current == current_original + + +def test_merge_table_reports_successful_decisions_in_page_order() -> None: + """验证成功诊断保留原标记行为,并按原倒序检测顺序报告。""" + tables = [_table(index) for index in range(3)] + pages = [_page(index, [table]) for index, table in enumerate(tables)] + decisions: list[TableMergeDecision] = [] + + merge_table(pages, on_decision=decisions.append) + + assert [table.get("continues_prev") for table in tables] == [None, True, True] + assert [(item.previous_page_idx, item.current_page_idx) for item in decisions] == [(1, 2), (0, 1)] + assert all(item.mergeable and item.rejection is None for item in decisions) + assert [(item.previous_table_index, item.current_table_index) for item in decisions] == [(1, 2), (0, 1)] + + +@pytest.mark.parametrize( + ("previous", "current", "expected_rejection"), + [ + ( + _table(0), + _table(1, children=[_caption("Table 1"), _table_body(index=1)]), + TableMergeRejection.CONTINUATION_CAPTION_REQUIRED, + ), + ( + _table(0, children=[_table_body(), _footnote("one", index=1), _footnote("two", index=2)]), + _table(1, children=[_caption("Table 1 (continued)"), _table_body(index=1)]), + TableMergeRejection.MULTIPLE_PREVIOUS_FOOTNOTES, + ), + ( + _table(0, children=[_table_body(), _footnote("old")]), + _table(1), + TableMergeRejection.PREVIOUS_FOOTNOTES_REQUIRE_CAPTION, + ), + ( + _table(0), + _table(1, bbox=[0.06, 0.1, 0.94, 0.9]), + TableMergeRejection.INCOMPATIBLE_TABLE_WIDTHS, + ), + ( + _table(0, "
H
AB
"), + _table(1, "
C
"), + TableMergeRejection.INCOMPATIBLE_TABLE_STRUCTURE, + ), + ], +) +def test_merge_table_diagnoses_rejected_table_rules( + previous: dict[str, Any], + current: dict[str, Any], + expected_rejection: TableMergeRejection, +) -> None: + """验证题注、脚注、宽度和列结构拒绝原因。""" + previous = deepcopy(previous) + current = deepcopy(current) + decisions: list[TableMergeDecision] = [] + + merge_table([_page(0, [previous]), _page(1, [current])], on_decision=decisions.append) + + assert "continues_prev" not in current + assert len(decisions) == 1 + assert decisions[0].mergeable is False + assert decisions[0].rejection is expected_rejection + + +@pytest.mark.parametrize( + ("pages", "expected_rejection", "expected_blocker_page", "expected_blocker_type"), + [ + ( + [_page(0, [_table(0), _noise(BlockType.TEXT)]), _page(1, [_table(1)])], + TableMergeRejection.BOUNDARY_BLOCKED, + 0, + BlockType.TEXT, + ), + ( + [_page(0, []), _page(1, [_table(1)])], + TableMergeRejection.BOUNDARY_TABLE_MISSING, + None, + None, + ), + ( + [_page(0, [_table(0)]), _page(2, [_table(1)])], + TableMergeRejection.NON_CONSECUTIVE_PAGES, + None, + None, + ), + ( + ["invalid", _page(1, [_table(1)])], + TableMergeRejection.INVALID_PAGE, + None, + None, + ), + ( + [_page(0, [_table(0)]), _page(1, [_table(1, "
")])], + TableMergeRejection.INVALID_TABLE_STRUCTURE, + None, + None, + ), + ], +) +def test_merge_table_diagnoses_page_and_boundary_rejections( + pages: list[dict[str, Any]], + expected_rejection: TableMergeRejection, + expected_blocker_page: int | None, + expected_blocker_type: str | None, +) -> None: + """验证页面、边界与表格结构拒绝原因及阻断位置。""" + pages = deepcopy(pages) + decisions: list[TableMergeDecision] = [] + + merge_table(pages, on_decision=decisions.append) + + assert len(decisions) == 1 + decision = decisions[0] + assert decision.rejection is expected_rejection + assert decision.blocker_page_idx == expected_blocker_page + assert decision.blocker_block_type == expected_blocker_type + + +def test_merge_table_diagnoses_preserved_pdf_structure() -> None: + """验证原生PDF结构块的保留决策可诊断。""" + current = _table(1) + current["_pdf_structure"] = {} + decisions: list[TableMergeDecision] = [] + + merge_table([_page(0, [_table(0)]), _page(1, [current])], on_decision=decisions.append) + + assert len(decisions) == 1 + assert decisions[0].rejection is TableMergeRejection.PDF_STRUCTURE_PRESERVED + + +def test_merge_table_rejects_invalid_diagnostic_callback_before_mutation() -> None: + """验证非法诊断回调在校验阶段拒绝,不清理已有标记。""" + current = _table(1) + current["continues_prev"] = True + pages = [_page(0, [_table(0)]), _page(1, [current])] + + with pytest.raises(TypeError, match="on_decision"): + merge_table(pages, on_decision="diagnostics") + + assert current["continues_prev"] is True