Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions docs/JSON_PROTOCOL.md
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,20 @@ Structured Content preserves the optional `anchor` in its `captions` and
`footnotes` entries; their `content` remains plain rendered annotation content
without an injected target tag.

## Cross-page table detection diagnostics

`merge_table()` accepts an optional `on_decision` callback for callers that need to
explain why a page pair was not marked as a continuation. Each immutable
`TableMergeDecision` contains the two page indices, available table block indices,
and a `TableMergeRejection` reason. `decision.mergeable` is true only when the
existing continuation checks accept the pair. Boundary rejections also identify the
blocking page and block type.

The callback reports detection decisions in the existing reverse page order. It does
not report whether a later renderer can merge the table content, and it does not
include table text. Omitting the callback preserves the existing behavior and output;
no document schema or serialized diagnostic fields are added.

## Optional PDF layout geometry

Native PDF analysis adds `extensions.docvortex_layout` and deterministic
Expand Down
4 changes: 3 additions & 1 deletion src/docvortex/content/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@
from .content import merge_table_content
from .document import merge_table
from .html import build_row_rendered_cell_segments, build_table_state_from_html, calculate_row_rendered_segments
from .models import RenderedCellSegment
from .models import RenderedCellSegment, TableMergeDecision, TableMergeRejection
from .structure import can_merge_by_structure, detect_table_headers
from .structure import _expand_header_count_by_rowspan as expand_header_count_by_rowspan

__all__ = [
"RenderedCellSegment",
"TableMergeDecision",
"TableMergeRejection",
"merge_table",
"merge_table_content",
"build_table_state_from_html",
Expand Down
158 changes: 144 additions & 14 deletions src/docvortex/content/table/document.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@

from __future__ import annotations

from typing import Any
from typing import Any, Callable

from ...schema import MERGE_TRANSPARENT_BLOCK_TYPES, BlockType

from .blocks import _get_or_create_table_state
from .models import BlockDict, PageInfoDict, TableMergeState
from .structure import can_merge_tables
from .models import BlockDict, PageInfoDict, TableMergeDecision, TableMergeRejection, TableMergeState
from .structure import _table_merge_rejection_reason

TABLE_BOUNDARY_IGNORED_TYPES = set(MERGE_TRANSPARENT_BLOCK_TYPES)

Expand Down Expand Up @@ -36,19 +36,29 @@ def _clear_nested_continuation_markers(block: BlockDict) -> None:
_clear_nested_continuation_markers(child)


def _find_boundary_table(blocks: list[Any], *, from_end: bool) -> BlockDict | None:
"""从页边界扫描 table;噪声块可跳过,其他语义块立即阻断。"""
def _find_boundary_table_with_blocker(
blocks: list[Any],
*,
from_end: bool,
) -> tuple[BlockDict | None, str | None]:
"""返回边界表格,或首个阻断块的类型。"""
ordered_blocks = reversed(blocks) if from_end else iter(blocks)
for block in ordered_blocks:
if not isinstance(block, dict):
return None
return None, type(block).__name__
block_type = block.get("type")
if block_type in TABLE_BOUNDARY_IGNORED_TYPES:
continue
if block_type == BlockType.TABLE:
return block
return None
return None
return block, None
return None, block_type if isinstance(block_type, str) else type(block_type).__name__
return None, None


def _find_boundary_table(blocks: list[Any], *, from_end: bool) -> BlockDict | None:
"""从页边界扫描 table;噪声块可跳过,其他语义块立即阻断。"""
table_block, _ = _find_boundary_table_with_blocker(blocks, from_end=from_end)
return table_block


def _is_consecutive_page_pair(previous_page: PageInfoDict, current_page: PageInfoDict) -> bool:
Expand All @@ -58,8 +68,61 @@ def _is_consecutive_page_pair(previous_page: PageInfoDict, current_page: PageInf
return type(previous_page_idx) is int and type(current_page_idx) is int and current_page_idx == previous_page_idx + 1


def merge_table(page_info_list: list[PageInfoDict]) -> None:
"""倒序识别连续页边界表格,并只在后表写入延续标记。"""
def _page_index(page: Any) -> int | None:
"""读取诊断使用的合法页索引,非法页不抛出额外异常。"""
if not isinstance(page, dict):
return None
page_idx = page.get("page_idx")
return page_idx if type(page_idx) is int else None


def _table_index(table_block: BlockDict | None) -> int | None:
"""读取诊断使用的表格块索引。"""
if not isinstance(table_block, dict):
return None
index = table_block.get("index")
return index if type(index) is int else None


def _report_decision(
on_decision: Callable[[TableMergeDecision], None] | None,
*,
previous_page: Any,
current_page: Any,
rejection: TableMergeRejection | None,
previous_table: BlockDict | None = None,
current_table: BlockDict | None = None,
blocker_page: Any = None,
blocker_block_type: str | None = None,
) -> None:
"""仅在调用方请求时构造并发送不可变诊断。"""
if on_decision is None:
return
on_decision(
TableMergeDecision(
previous_page_idx=_page_index(previous_page),
current_page_idx=_page_index(current_page),
previous_table_index=_table_index(previous_table),
current_table_index=_table_index(current_table),
rejection=rejection,
blocker_page_idx=_page_index(blocker_page),
blocker_block_type=blocker_block_type,
)
)


def merge_table(
page_info_list: list[PageInfoDict],
*,
on_decision: Callable[[TableMergeDecision], None] | None = None,
) -> None:
"""倒序识别连续页边界表格,并只在后表写入延续标记。

``on_decision`` 可选接收不含正文的检测诊断,不改变默认标记行为。
诊断只覆盖延续检测,不承诺后续渲染能够合并表格内容。
"""
if on_decision is not None and not callable(on_decision):
raise TypeError("on_decision must be callable or None")
if not isinstance(page_info_list, list):
return

Expand All @@ -79,28 +142,95 @@ def merge_table(page_info_list: list[PageInfoDict]) -> None:
current_page = page_info_list[page_position]
previous_page = page_info_list[page_position - 1]
if not isinstance(current_page, dict) or not isinstance(previous_page, dict):
_report_decision(
on_decision,
previous_page=previous_page,
current_page=current_page,
rejection=TableMergeRejection.INVALID_PAGE,
)
continue
if not _is_consecutive_page_pair(previous_page, current_page):
_report_decision(
on_decision,
previous_page=previous_page,
current_page=current_page,
rejection=TableMergeRejection.NON_CONSECUTIVE_PAGES,
)
continue

current_blocks = current_page.get("blocks")
previous_blocks = previous_page.get("blocks")
if not isinstance(current_blocks, list) or not isinstance(previous_blocks, list):
_report_decision(
on_decision,
previous_page=previous_page,
current_page=current_page,
rejection=TableMergeRejection.INVALID_PAGE,
)
continue

current_table_block = _find_boundary_table(current_blocks, from_end=False)
previous_table_block = _find_boundary_table(previous_blocks, from_end=True)
current_table_block, current_blocker_type = _find_boundary_table_with_blocker(current_blocks, from_end=False)
previous_table_block, previous_blocker_type = _find_boundary_table_with_blocker(previous_blocks, from_end=True)
if current_table_block is None or previous_table_block is None:
current_is_blocked = current_table_block is None and current_blocker_type is not None
previous_is_blocked = previous_table_block is None and previous_blocker_type is not None
_report_decision(
on_decision,
previous_page=previous_page,
current_page=current_page,
rejection=(
TableMergeRejection.BOUNDARY_BLOCKED
if current_is_blocked or previous_is_blocked
else TableMergeRejection.BOUNDARY_TABLE_MISSING
),
previous_table=previous_table_block,
current_table=current_table_block,
blocker_page=current_page if current_is_blocked else previous_page if previous_is_blocked else None,
blocker_block_type=current_blocker_type if current_is_blocked else previous_blocker_type,
)
continue
if "_pdf_structure" in current_table_block or "_pdf_structure" in previous_table_block:
_report_decision(
on_decision,
previous_page=previous_page,
current_page=current_page,
rejection=TableMergeRejection.PDF_STRUCTURE_PRESERVED,
previous_table=previous_table_block,
current_table=current_table_block,
)
continue

current_state = _get_or_create_table_state(current_table_block, state_cache)
previous_state = _get_or_create_table_state(previous_table_block, state_cache)
if current_state is None or previous_state is None:
_report_decision(
on_decision,
previous_page=previous_page,
current_page=current_page,
rejection=TableMergeRejection.INVALID_TABLE_STRUCTURE,
previous_table=previous_table_block,
current_table=current_table_block,
)
continue

if not can_merge_tables(current_state, previous_state):
rejection = _table_merge_rejection_reason(current_state, previous_state)
if rejection is not None:
_report_decision(
on_decision,
previous_page=previous_page,
current_page=current_page,
rejection=rejection,
previous_table=previous_table_block,
current_table=current_table_block,
)
continue

current_table_block["continues_prev"] = True
_report_decision(
on_decision,
previous_page=previous_page,
current_page=current_page,
rejection=None,
previous_table=previous_table_block,
current_table=current_table_block,
)
35 changes: 35 additions & 0 deletions src/docvortex/content/table/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from __future__ import annotations

from dataclasses import dataclass
from enum import Enum
from typing import Any, TypeAlias

MAX_HEADER_ROWS = 5
Expand All @@ -12,6 +13,40 @@
CalculationBBox: TypeAlias = tuple[int, int, int, int]


class TableMergeRejection(str, Enum):
"""跨页表格延续检测的拒绝原因。"""

INVALID_PAGE = "invalid_page"
NON_CONSECUTIVE_PAGES = "non_consecutive_pages"
BOUNDARY_TABLE_MISSING = "boundary_table_missing"
BOUNDARY_BLOCKED = "boundary_blocked"
PDF_STRUCTURE_PRESERVED = "pdf_structure_preserved"
INVALID_TABLE_STRUCTURE = "invalid_table_structure"
CONTINUATION_CAPTION_REQUIRED = "continuation_caption_required"
MULTIPLE_PREVIOUS_FOOTNOTES = "multiple_previous_footnotes"
PREVIOUS_FOOTNOTES_REQUIRE_CAPTION = "previous_footnotes_require_caption"
INCOMPATIBLE_TABLE_WIDTHS = "incompatible_table_widths"
INCOMPATIBLE_TABLE_STRUCTURE = "incompatible_table_structure"


@dataclass(frozen=True, slots=True)
class TableMergeDecision:
"""记录一次相邻页表格检测的结果,不包含论文正文。"""

previous_page_idx: int | None
current_page_idx: int | None
previous_table_index: int | None
current_table_index: int | None
rejection: TableMergeRejection | None
blocker_page_idx: int | None = None
blocker_block_type: str | None = None

@property
def mergeable(self) -> bool:
"""返回相邻页表格是否通过延续检测。"""
return self.rejection is None


@dataclass
class RowMetrics:
"""记录单行的有效列、实际列和视觉列指标。"""
Expand Down
28 changes: 19 additions & 9 deletions src/docvortex/content/table/structure.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

from .blocks import _bbox_for_calculation, _is_continuation_caption, _is_post_table_non_continuation_caption, _table_children
from .html import _colspan, _rowspan, calculate_row_rendered_segments
from .models import MAX_HEADER_ROWS, TableMergeState
from .models import MAX_HEADER_ROWS, TableMergeRejection, TableMergeState


def detect_table_headers(
Expand Down Expand Up @@ -153,13 +153,16 @@ def _table_widths_are_compatible(current_bbox: Any, previous_bbox: Any) -> bool:
return min_width > 0 and abs(current_width - previous_width) / min_width < 0.1


def can_merge_tables(current_state: TableMergeState, previous_state: TableMergeState) -> bool:
"""根据 dict 表格的辅助文本、宽度和 HTML 结构判断是否可合并。"""
def _table_merge_rejection_reason(
current_state: TableMergeState,
previous_state: TableMergeState,
) -> TableMergeRejection | None:
"""按原判定顺序返回首个拒绝原因,成功时返回空。"""
current_table_block = current_state.owner_block
previous_table_block = previous_state.owner_block

if not isinstance(previous_table_block, dict) or not isinstance(current_table_block, dict):
return False
return TableMergeRejection.INVALID_TABLE_STRUCTURE

previous_children = _table_children(previous_table_block)
current_children = _table_children(current_table_block)
Expand All @@ -172,17 +175,24 @@ def can_merge_tables(current_state: TableMergeState, previous_state: TableMergeS
has_continuation_marker = any(_is_continuation_caption(block) for block in merge_caption_blocks)

if not has_continuation_marker:
return False
return TableMergeRejection.CONTINUATION_CAPTION_REQUIRED

if footnote_count > 1:
return False
return TableMergeRejection.MULTIPLE_PREVIOUS_FOOTNOTES
elif footnote_count > 0:
return False
return TableMergeRejection.PREVIOUS_FOOTNOTES_REQUIRE_CAPTION

if not _table_widths_are_compatible(current_table_block.get("bbox"), previous_table_block.get("bbox")):
return False
return TableMergeRejection.INCOMPATIBLE_TABLE_WIDTHS

if not can_merge_by_structure(current_state, previous_state):
return TableMergeRejection.INCOMPATIBLE_TABLE_STRUCTURE
return None

return can_merge_by_structure(current_state, previous_state)

def can_merge_tables(current_state: TableMergeState, previous_state: TableMergeState) -> bool:
"""根据 dict 表格的辅助文本、宽度和 HTML 结构判断是否可合并。"""
return _table_merge_rejection_reason(current_state, previous_state) is None


def check_rows_match(previous_state: TableMergeState, current_state: TableMergeState) -> bool:
Expand Down
2 changes: 2 additions & 0 deletions src/docvortex/public_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,8 @@
"docvortex.content.table": (
"build_row_rendered_cell_segments",
"RenderedCellSegment",
"TableMergeDecision",
"TableMergeRejection",
"build_table_state_from_html",
"calculate_row_rendered_segments",
"can_merge_by_structure",
Expand Down
Loading