diff --git a/CHANGELOG.md b/CHANGELOG.md index 71f37d5..d5e45fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ the project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html ## [Unreleased] +## [0.4.4] — 2026-08-13 + +### Fixed + +- **Duplicate header labels crashed the multipage merge** (`merger.py`). + Extracted headers that repeat a label — e.g. SEC 13F voting-authority + triplets where TableFormer emits `COLUMN 8` three times — made + `_build_generic_merged_table` raise `ValueError: Reindexing only valid + with uniquely valued Index objects`, because `pd.concat` cannot align + fragments on a non-unique column Index. Downstream consumers that + fail-soft on the error silently kept a degraded table set for the whole + document. Fragments are now merged under positionally deduped labels + (collision-safe against pre-existing `X.1`-style names) and the original + duplicated labels are restored on the merged output, matching how + single-fragment tables pass through untouched. + ## [0.4.3] — 2026-06-11 ### Fixed diff --git a/README.md b/README.md index 5faaa74..44c6e75 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,8 @@ from table_stitcher import stitch_tables converter = DocumentConverter() doc = converter.convert("report.pdf").document -doc = stitch_tables(doc) # merged tables; ready for - # export_to_markdown() / HTML / LLM +doc = stitch_tables(doc) # merged tables; ready for +# export_to_markdown() / HTML / LLM ``` `stitch_tables()` mutates `doc` in place and returns the same object. If you @@ -94,10 +94,10 @@ Runnable end-to-end scripts live in [`examples/`](examples/): from table_stitcher import stitch_tables, MultiPageConfig config = MultiPageConfig( - max_page_gap=1, # Only merge tables on consecutive pages - max_width_difference=2, # Column count tolerance - header_sim_strict=0.6, # Threshold for repeated header detection - stitch_separator="\n", # Join character for split content + max_page_gap=1, # Only merge tables on consecutive pages + max_width_difference=2, # Column count tolerance + header_sim_strict=0.6, # Threshold for repeated header detection + stitch_separator="\n", # Join character for split content ) doc = stitch_tables(doc, config=config) @@ -110,6 +110,7 @@ from typing import Any, List from table_stitcher import TableStitcher, MultiPageConfig, TableMeta, LogicalTable from table_stitcher.adapters.base import TableStitcherAdapter + class MyParserAdapter: def extract(self, doc, cfg: MultiPageConfig) -> List[TableMeta]: """Read tables from your document format into TableMeta objects.""" @@ -119,6 +120,7 @@ class MyParserAdapter: """Write merged results back into your document format.""" ... + stitcher = TableStitcher(adapter=MyParserAdapter()) doc = stitcher.stitch(doc) ``` @@ -220,7 +222,13 @@ from typing import Any, List import pandas as pd from table_stitcher import TableStitcher, MultiPageConfig, TableMeta, LogicalTable from table_stitcher.adapters.base import TableStitcherAdapter -from table_stitcher.merger import tokenize, normalize_col_name, is_numeric_like_colnames, first_row_has_number +from table_stitcher.merger import ( + tokenize, + normalize_col_name, + is_numeric_like_colnames, + first_row_has_number, +) + class MyParserAdapter: def extract(self, doc: Any, cfg: MultiPageConfig) -> List[TableMeta]: @@ -243,32 +251,32 @@ class MyParserAdapter: # 4. Tokenize first row (fallback similarity signal) first_row_tokens = set() if df.shape[0] > 0: - first_row_tokens = tokenize( - " ".join(str(x) for x in df.iloc[0].tolist()) - ) + first_row_tokens = tokenize(" ".join(str(x) for x in df.iloc[0].tolist())) # 5. Classify: is_headerless, is_header_orphan, is_data_orphan raw_columns = [str(c) for c in df.columns] - is_headerless = df.attrs.get('is_headerless', False) - - tables_meta.append(TableMeta( - idx=idx, - df=df, - start_page=start_page, - pages=pages, - width=df.shape[1], - header_tokens=header_tokens, - first_row_tokens=first_row_tokens, - raw_columns=raw_columns, - vert_center=None, # Set if bbox available - vert_top=None, # Normalized 0-1, 0=top of page - vert_bottom=None, # Normalized 0-1, 1=bottom of page - is_header_orphan=False, # True if headers-only, no/few data rows - is_data_orphan=False, # True if data-only, no real headers - numeric_like_cols=is_numeric_like_colnames(raw_columns), - row_count=df.shape[0], - is_headerless=is_headerless, - )) + is_headerless = df.attrs.get("is_headerless", False) + + tables_meta.append( + TableMeta( + idx=idx, + df=df, + start_page=start_page, + pages=pages, + width=df.shape[1], + header_tokens=header_tokens, + first_row_tokens=first_row_tokens, + raw_columns=raw_columns, + vert_center=None, # Set if bbox available + vert_top=None, # Normalized 0-1, 0=top of page + vert_bottom=None, # Normalized 0-1, 1=bottom of page + is_header_orphan=False, # True if headers-only, no/few data rows + is_data_orphan=False, # True if data-only, no real headers + numeric_like_cols=is_numeric_like_colnames(raw_columns), + row_count=df.shape[0], + is_headerless=is_headerless, + ) + ) return tables_meta def inject(self, doc: Any, logical_tables: List[LogicalTable]) -> Any: @@ -287,6 +295,7 @@ class MyParserAdapter: return doc + # Use it: stitcher = TableStitcher(adapter=MyParserAdapter()) doc = stitcher.stitch(doc) @@ -336,6 +345,7 @@ except StitchingError as e: ```python import logging + logging.getLogger("table_stitcher").setLevel(logging.INFO) ``` diff --git a/pyproject.toml b/pyproject.toml index 058603d..8b86ce1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "table-stitcher" -version = "0.4.3" +version = "0.4.4" description = "Reassemble tables split across page boundaries in PDF extraction" readme = "README.md" license = "MIT" diff --git a/src/table_stitcher/adapters/README.md b/src/table_stitcher/adapters/README.md index 0be1847..397f565 100644 --- a/src/table_stitcher/adapters/README.md +++ b/src/table_stitcher/adapters/README.md @@ -104,9 +104,9 @@ A few structural constants live at the top of `docling.py` rather than in `MultiPageConfig`: ```python -_MAX_HEADER_CELL_LEN = 30 # header cells typically short; data cells longer -_DATA_PATTERNS # regex list for "this cell is data, not header" -_AUTO_COLNAME_RE # "Column_N" / "Unnamed: N" parser placeholders +_MAX_HEADER_CELL_LEN = 30 # header cells typically short; data cells longer +_DATA_PATTERNS # regex list for "this cell is data, not header" +_AUTO_COLNAME_RE # "Column_N" / "Unnamed: N" parser placeholders ``` These are **adapter-intrinsic** — tuning them changes how the adapter diff --git a/src/table_stitcher/merger.py b/src/table_stitcher/merger.py index 6298461..0a5c29b 100644 --- a/src/table_stitcher/merger.py +++ b/src/table_stitcher/merger.py @@ -619,13 +619,34 @@ def _build_orphan_merged_table( ) +def _dedupe_labels(labels: list[str]) -> list[str]: + """Make labels unique by suffixing repeats (``B, B`` -> ``B, B.1``).""" + used: set[str] = set() + counts: dict[str, int] = {} + out: list[str] = [] + for label in labels: + candidate = label + while candidate in used: + counts[label] = counts.get(label, 0) + 1 + candidate = f"{label}.{counts[label]}" + used.add(candidate) + out.append(candidate) + return out + + def _build_generic_merged_table( members: list[int], meta_by_idx: dict[int, TableMeta], cfg: MultiPageConfig ) -> tuple[pd.DataFrame, set[int], list[str]]: """Build merged table for the general case.""" base = meta_by_idx[members[0]] merged_df = base.df.copy() - canonical_cols = [str(c) for c in base.df.columns] + # Duplicate header labels are normal in the wild (rowspan/colspan + # parsers, 13F voting-authority triplets), but pd.concat cannot align + # frames on a non-unique column Index. Merge under deduped labels and + # restore the originals on the way out. + original_cols = [str(c) for c in base.df.columns] + canonical_cols = _dedupe_labels(original_cols) + merged_df.columns = canonical_cols merged_pages = set(base.pages) warnings: list[str] = [] prev = base @@ -648,6 +669,7 @@ def _build_generic_merged_table( merged_pages.update(m.pages) prev = m + merged_df.columns = original_cols + [str(c) for c in merged_df.columns[len(original_cols) :]] return merged_df, merged_pages, warnings diff --git a/tests/test_merger.py b/tests/test_merger.py index 0ef5a18..b40ddb7 100644 --- a/tests/test_merger.py +++ b/tests/test_merger.py @@ -228,6 +228,50 @@ def test_invalid_policy_raises_for_direct_core_calls(self): align_dataframe_to_header(df, ["A"], meta, cfg) +class TestDuplicateHeaderLabels: + def test_merge_survives_duplicate_header_labels(self): + """ + Extracted headers repeating a label (e.g. 13F voting-authority + triplets emitting 'COLUMN 8' x3) must not crash the multipage + merge: pd.concat cannot align frames on a non-unique column Index. + """ + cols = ["A", "B", "B"] + df1 = pd.DataFrame([["1", "2", "3"]], columns=cols) + df2 = pd.DataFrame([["4", "5", "6"]], columns=cols) + metas = [ + _make_meta(idx=0, df=df1, start_page=1), + _make_meta(idx=1, df=df2, start_page=2), + ] + results = merge_multipage_tables(metas, MultiPageConfig()) + assert len(results) == 1 + assert results[0].df.shape == (2, 3) + assert results[0].df.iloc[1].tolist() == ["4", "5", "6"] + # Original (duplicated) labels are preserved in the output, matching + # how single-fragment tables pass through untouched. + assert list(results[0].df.columns) == cols + + def test_duplicate_labels_with_wider_continuation(self): + cols = ["A", "B", "B"] + df1 = pd.DataFrame([["1", "2", "3"]], columns=cols) + df2 = pd.DataFrame([["4", "5", "6", "7"]], columns=cols + ["C"]) + metas = [ + _make_meta(idx=0, df=df1, start_page=1), + _make_meta(idx=1, df=df2, start_page=2), + ] + results = merge_multipage_tables(metas, MultiPageConfig()) + assert len(results) == 1 + assert results[0].df.shape == (2, 4) + assert results[0].df.iloc[1, 3] == "7" + assert list(results[0].df.columns)[:3] == cols + + def test_dedupe_labels_avoids_existing_suffix_collision(self): + from table_stitcher.merger import _dedupe_labels + + out = _dedupe_labels(["X", "X", "X.1"]) + assert len(set(out)) == 3 + assert out[0] == "X" + + class TestMergeTrace: def test_logical_table_explains_merge_reason_and_signals(self): df = pd.DataFrame({"Name": ["Alice"], "Age": ["30"]})