From 8c34dbfb51fc06c10cf7f1d3e8bd3e83fe6d4744 Mon Sep 17 00:00:00 2001 From: airmang <38392618+airmang@users.noreply.github.com> Date: Sat, 4 Jul 2026 23:02:00 +0900 Subject: [PATCH] feat(ingest): add MarkItDown-style document ingest gateway --- README.md | 1 + src/hwpx/__init__.py | 18 +++ src/hwpx/ingest/__init__.py | 26 ++++ src/hwpx/ingest/base.py | 230 ++++++++++++++++++++++++++++++ src/hwpx/ingest/hwpx_converter.py | 127 +++++++++++++++++ tests/test_ingest.py | 65 +++++++++ 6 files changed, 467 insertions(+) create mode 100644 src/hwpx/ingest/__init__.py create mode 100644 src/hwpx/ingest/base.py create mode 100644 src/hwpx/ingest/hwpx_converter.py create mode 100644 tests/test_ingest.py diff --git a/README.md b/README.md index 2d81a96..f1a2d16 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ new.save_to_path("์ƒˆ๋ฌธ์„œ.hwpx") ### ๐Ÿ” ์ฝ๊ธฐ ยท ์ถ”์ถœ - ํ…์ŠคํŠธ/HTML/Markdown ๋‚ด๋ณด๋‚ด๊ธฐ โ€” `export_text()` ยท `export_html()` ยท `export_markdown()` - **ํ’๋ถ€ํ•œ Markdown** โ€” `export_rich_markdown()`์€ ์ธ๋ผ์ธ ์„œ์‹(`**๊ตต๊ฒŒ**`ยท`*๊ธฐ์šธ์ž„*`ยท`~~์ทจ์†Œ์„ ~~`), ์ค‘์ฒฉ ํ‘œ(colspan/rowspan ์•ˆ์ „), ๋„ํ˜• ํ…์ŠคํŠธ, ์ด๋ฏธ์ง€, ๊ฐ์ฃผ/๋ฏธ์ฃผ, ํ•˜์ดํผ๋งํฌ, ์ œ๋ชฉ(`#`/`##`) ์ž๋™ ๊ฐ์ง€๊นŒ์ง€ ๋ณด์กด +- **๋ฌธ์„œ ingest ๊ฒŒ์ดํŠธ์›จ์ด** โ€” `hwpx.ingest.DocumentIngestor`๊ฐ€ HWPX๋ฅผ ๊ฐ์ง€ํ•ด rich Markdown๊ณผ ์„น์…˜/ํ‘œ ๋ฉ”ํƒ€๋ฐ์ดํ„ฐ๋กœ ์ •๊ทœํ™” - `TextExtractor` / `ObjectFinder` โ€” ์„น์…˜ยท๋ฌธ๋‹จ ์ˆœํšŒ, ํƒœ๊ทธยท์†์„ฑยทXPath๋กœ ๊ฐ์ฒด ํƒ์ƒ‰ (`hp:tab`์€ `\t`๋กœ ๋ณด์กด, roundtrip ์•ˆ์ „) ```python diff --git a/src/hwpx/__init__.py b/src/hwpx/__init__.py index 16447c7..1a62d16 100644 --- a/src/hwpx/__init__.py +++ b/src/hwpx/__init__.py @@ -75,6 +75,16 @@ def __getattr__(name: str) -> object: PreviewPage, render_layout_preview, ) +from .ingest import ( + ConversionAttempt, + DocumentConverter, + DocumentIngestError, + DocumentIngestResult, + DocumentIngestor, + DocumentSourceInfo, + HwpxMarkdownConverter, + UnsupportedDocumentFormat, +) from .patch import ( BytePreservingPatchResult, ParagraphTextPatch, @@ -123,8 +133,14 @@ def __getattr__(name: str) -> object: "DEFAULT_STYLE_PRESET", "DOCUMENT_PLAN_SCHEMA_VERSION", "get_document_plan_schema", + "ConversionAttempt", "DocumentBlock", + "DocumentConverter", + "DocumentIngestError", + "DocumentIngestResult", + "DocumentIngestor", "DocumentPlan", + "DocumentSourceInfo", "DocumentStylePreset", "EditorOpenSafetyReport", "ParagraphInfo", @@ -141,6 +157,7 @@ def __getattr__(name: str) -> object: "TEMPLATE_FORMFIT_PLAN_SCHEMA_VERSION", "TextExtractor", "FoundElement", + "HwpxMarkdownConverter", "ObjectFinder", "OFFICIAL_DOCUMENT_STYLE_REPORT_VERSION", "DOC_DIFF_REPORT_VERSION", @@ -150,6 +167,7 @@ def __getattr__(name: str) -> object: "STYLE_PROFILE_SCHEMA_VERSION", "TEMPLATE_REGISTRY_SCHEMA_VERSION", "TABLE_COMPUTE_REPORT_VERSION", + "UnsupportedDocumentFormat", "apply_style_profile_to_plan", "build_comparison_table_plan", "build_image_grid", diff --git a/src/hwpx/ingest/__init__.py b/src/hwpx/ingest/__init__.py new file mode 100644 index 0000000..0ffc432 --- /dev/null +++ b/src/hwpx/ingest/__init__.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Document ingest primitives for HWPX-first Markdown conversion.""" + +from .base import ( + ConversionAttempt, + DocumentConverter, + DocumentIngestError, + DocumentIngestResult, + DocumentIngestor, + DocumentSourceInfo, + UnsupportedDocumentFormat, + normalize_markdown, +) +from .hwpx_converter import HwpxMarkdownConverter + +__all__ = [ + "ConversionAttempt", + "DocumentConverter", + "DocumentIngestError", + "DocumentIngestResult", + "DocumentIngestor", + "DocumentSourceInfo", + "HwpxMarkdownConverter", + "UnsupportedDocumentFormat", + "normalize_markdown", +] diff --git a/src/hwpx/ingest/base.py b/src/hwpx/ingest/base.py new file mode 100644 index 0000000..237b180 --- /dev/null +++ b/src/hwpx/ingest/base.py @@ -0,0 +1,230 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Small MarkItDown-style ingest dispatcher.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from io import BytesIO, TextIOBase +import mimetypes +from pathlib import Path +import re +from typing import Any, BinaryIO, Iterable, Protocol + + +@dataclass(frozen=True, kw_only=True) +class DocumentSourceInfo: + """Metadata known or inferred about an input document.""" + + mimetype: str | None = None + extension: str | None = None + charset: str | None = None + filename: str | None = None + local_path: str | None = None + url: str | None = None + + def copy_and_update(self, *others: "DocumentSourceInfo", **kwargs: Any) -> "DocumentSourceInfo": + data = asdict(self) + for other in others: + data.update({key: val for key, val in asdict(other).items() if val is not None}) + data.update({key: val for key, val in kwargs.items() if val is not None}) + return DocumentSourceInfo(**data) + + +@dataclass(frozen=True, kw_only=True) +class ConversionAttempt: + """One converter decision or failure.""" + + converter: str + accepted: bool + error_type: str | None = None + message: str | None = None + + def as_dict(self) -> dict[str, Any]: + return {key: val for key, val in asdict(self).items() if val is not None} + + +@dataclass(kw_only=True) +class DocumentIngestResult: + """Normalized Markdown plus structured conversion metadata.""" + + markdown: str + source_info: DocumentSourceInfo + source_format: str + engine: str + engine_version: str | None = None + title: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + sections: list[dict[str, Any]] = field(default_factory=list) + tables: list[dict[str, Any]] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + attempts: list[ConversionAttempt] = field(default_factory=list) + lossiness: str = "unknown" + + @property + def text_content(self) -> str: + """Compatibility alias matching common Markdown conversion APIs.""" + + return self.markdown + + def as_dict(self) -> dict[str, Any]: + return { + "markdown": self.markdown, + "sourceInfo": asdict(self.source_info), + "sourceFormat": self.source_format, + "engine": self.engine, + "engineVersion": self.engine_version, + "title": self.title, + "metadata": self.metadata, + "sections": self.sections, + "tables": self.tables, + "warnings": self.warnings, + "attempts": [attempt.as_dict() for attempt in self.attempts], + "lossiness": self.lossiness, + } + + +class DocumentIngestError(RuntimeError): + """Base class for ingest failures with converter-attempt diagnostics.""" + + def __init__(self, message: str, *, attempts: Iterable[ConversionAttempt] = ()) -> None: + super().__init__(message) + self.attempts = list(attempts) + + def as_dict(self) -> dict[str, Any]: + return { + "error": type(self).__name__, + "message": str(self), + "attempts": [attempt.as_dict() for attempt in self.attempts], + } + + +class UnsupportedDocumentFormat(DocumentIngestError): + """Raised when no registered converter accepts the source.""" + + +class DocumentConverter(Protocol): + """Converter contract for a specific source family.""" + + name: str + + def accepts(self, file_stream: BinaryIO, source_info: DocumentSourceInfo) -> bool: + """Return True if this converter should attempt conversion.""" + + def convert( + self, + file_stream: BinaryIO, + source_info: DocumentSourceInfo, + **kwargs: Any, + ) -> DocumentIngestResult: + """Convert the source into normalized Markdown.""" + + +def normalize_markdown(markdown: str) -> str: + """Normalize whitespace in generated Markdown without changing content meaning.""" + + text = "\n".join(line.rstrip() for line in re.split(r"\r?\n", markdown or "")) + return re.sub(r"\n{3,}", "\n\n", text).strip() + + +class DocumentIngestor: + """Priority-ordered converter dispatcher.""" + + def __init__(self, converters: Iterable[DocumentConverter] | None = None) -> None: + self._converters: list[tuple[float, int, DocumentConverter]] = [] + self._sequence = 0 + if converters is not None: + for converter in converters: + self.register_converter(converter) + + @classmethod + def default(cls) -> "DocumentIngestor": + from .hwpx_converter import HwpxMarkdownConverter + + ingestor = cls() + ingestor.register_converter(HwpxMarkdownConverter(), priority=0.0) + return ingestor + + def register_converter(self, converter: DocumentConverter, *, priority: float = 0.0) -> None: + self._converters.append((priority, self._sequence, converter)) + self._sequence += 1 + + def convert( + self, + source: str | Path | bytes | BinaryIO, + *, + source_info: DocumentSourceInfo | None = None, + **kwargs: Any, + ) -> DocumentIngestResult: + stream, inferred = _source_to_stream(source) + info = inferred if source_info is None else inferred.copy_and_update(source_info) + attempts: list[ConversionAttempt] = [] + start_pos = stream.tell() + + for _priority, _seq, converter in sorted(self._converters, key=lambda item: (item[0], item[1])): + name = getattr(converter, "name", type(converter).__name__) + stream.seek(start_pos) + try: + accepted = bool(converter.accepts(stream, info)) + except Exception as exc: + attempts.append( + ConversionAttempt( + converter=name, + accepted=False, + error_type=type(exc).__name__, + message=str(exc), + ) + ) + continue + finally: + stream.seek(start_pos) + + attempts.append(ConversionAttempt(converter=name, accepted=accepted)) + if not accepted: + continue + + try: + result = converter.convert(stream, info, **kwargs) + except Exception as exc: + attempts[-1] = ConversionAttempt( + converter=name, + accepted=True, + error_type=type(exc).__name__, + message=str(exc), + ) + stream.seek(start_pos) + continue + result.markdown = normalize_markdown(result.markdown) + result.attempts = attempts + return result + + raise UnsupportedDocumentFormat("no registered converter accepted the document", attempts=attempts) + + +def _source_to_stream(source: str | Path | bytes | BinaryIO) -> tuple[BinaryIO, DocumentSourceInfo]: + if isinstance(source, bytes): + return BytesIO(source), DocumentSourceInfo() + if isinstance(source, Path): + return _path_to_stream(source) + if isinstance(source, str): + if source.startswith(("http:", "https:", "file:", "data:")): + raise UnsupportedDocumentFormat("URI sources are not implemented in python-hwpx ingest") + return _path_to_stream(Path(source)) + if hasattr(source, "read") and callable(source.read) and not isinstance(source, TextIOBase): + stream = source + if not stream.seekable(): + stream = BytesIO(stream.read()) + return stream, DocumentSourceInfo() + raise TypeError(f"unsupported document source type: {type(source).__name__}") + + +def _path_to_stream(path: Path) -> tuple[BinaryIO, DocumentSourceInfo]: + resolved = path.expanduser().resolve() + mimetype, charset = mimetypes.guess_type(str(resolved), strict=False) + info = DocumentSourceInfo( + mimetype=mimetype, + charset=charset, + extension=resolved.suffix.lower() or None, + filename=resolved.name, + local_path=str(resolved), + ) + return BytesIO(resolved.read_bytes()), info diff --git a/src/hwpx/ingest/hwpx_converter.py b/src/hwpx/ingest/hwpx_converter.py new file mode 100644 index 0000000..7020941 --- /dev/null +++ b/src/hwpx/ingest/hwpx_converter.py @@ -0,0 +1,127 @@ +# SPDX-License-Identifier: Apache-2.0 +"""HWPX ingest converter.""" + +from __future__ import annotations + +from importlib.metadata import PackageNotFoundError, version as package_version +from typing import Any, BinaryIO +from zipfile import BadZipFile, ZipFile + +from hwpx.document import HwpxDocument + +from .base import DocumentIngestResult, DocumentSourceInfo + + +class HwpxMarkdownConverter: + """Convert HWPX packages to rich Markdown and lightweight structure metadata.""" + + name = "HwpxMarkdownConverter" + + def accepts(self, file_stream: BinaryIO, source_info: DocumentSourceInfo) -> bool: + extension = (source_info.extension or "").lower() + if extension == ".hwpx": + return True + mimetype = (source_info.mimetype or "").lower() + if mimetype in {"application/hwp+zip", "application/x-hwp+zip"}: + return True + return _looks_like_hwpx_package(file_stream) + + def convert( + self, + file_stream: BinaryIO, + source_info: DocumentSourceInfo, + **kwargs: Any, + ) -> DocumentIngestResult: + start_pos = file_stream.tell() + data = file_stream.read() + file_stream.seek(start_pos) + + markdown_kwargs = dict(kwargs.pop("markdown_options", {}) or {}) + markdown_kwargs.setdefault("detect_headings", True) + + doc = HwpxDocument.open(data) + try: + markdown = doc.export_rich_markdown(**markdown_kwargs) + sections = _sections_payload(doc) + tables = _tables_payload(doc) + paragraph_count = len(list(doc.paragraphs)) + finally: + doc.close() + + return DocumentIngestResult( + markdown=markdown, + source_info=source_info, + source_format="hwpx", + engine="python-hwpx", + engine_version=_python_hwpx_version(), + metadata={ + "section_count": len(sections), + "paragraph_count": paragraph_count, + "table_count": len(tables), + "converter": self.name, + }, + sections=sections, + tables=tables, + lossiness="low", + ) + + +def _looks_like_hwpx_package(file_stream: BinaryIO) -> bool: + cur_pos = file_stream.tell() + try: + with ZipFile(file_stream) as archive: + names = set(archive.namelist()) + if "mimetype" in names: + try: + mimetype = archive.read("mimetype").decode("utf-8", "replace").strip() + if "hwp" in mimetype.lower() or "hwpx" in mimetype.lower(): + return True + except Exception: + pass + return any(name.startswith("Contents/section") and name.endswith(".xml") for name in names) + except (BadZipFile, OSError): + return False + finally: + file_stream.seek(cur_pos) + + +def _sections_payload(doc: HwpxDocument) -> list[dict[str, Any]]: + sections: list[dict[str, Any]] = [] + for index, section in enumerate(doc.sections): + paragraphs = list(section.paragraphs) + sections.append( + { + "index": index, + "paragraph_count": len(paragraphs), + "text_preview": "\n".join( + (paragraph.text or "").strip() + for paragraph in paragraphs[:5] + if (paragraph.text or "").strip() + )[:500], + } + ) + return sections + + +def _tables_payload(doc: HwpxDocument) -> list[dict[str, Any]]: + tables: list[dict[str, Any]] = [] + for paragraph_index, paragraph in enumerate(doc.paragraphs): + for table in paragraph.tables: + rows = [[cell.text for cell in row.cells] for row in table.rows] + tables.append( + { + "table_index": len(tables), + "paragraph_index": paragraph_index, + "rows": len(rows), + "cols": max((len(row) for row in rows), default=0), + "data": rows, + } + ) + return tables + + +def _python_hwpx_version() -> str: + try: + return package_version("python-hwpx") + except PackageNotFoundError: + return "0+unknown" diff --git a/tests/test_ingest.py b/tests/test_ingest.py new file mode 100644 index 0000000..b59d39d --- /dev/null +++ b/tests/test_ingest.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from io import BytesIO +from pathlib import Path + +import pytest + +from hwpx import HwpxDocument +from hwpx.ingest import DocumentIngestor, UnsupportedDocumentFormat, normalize_markdown + + +def _sample_hwpx(path: Path) -> Path: + doc = HwpxDocument.new() + doc.add_paragraph("2026ํ•™๋…„๋„ ์šด์˜ ๊ณ„ํš") + doc.add_paragraph("1. ์ถ”์ง„ ๋ชฉ์ ") + table = doc.add_table(2, 2) + table.set_cell_text(0, 0, "ํ•ญ๋ชฉ") + table.set_cell_text(0, 1, "๋‚ด์šฉ") + table.set_cell_text(1, 0, "๊ธฐ๊ฐ„") + table.set_cell_text(1, 1, "2026. 3.") + doc.save_to_path(path) + return path + + +def test_document_ingestor_converts_hwpx_to_rich_markdown(tmp_path: Path) -> None: + source = _sample_hwpx(tmp_path / "sample.hwpx") + + result = DocumentIngestor.default().convert(source) + + assert result.engine == "python-hwpx" + assert result.source_format == "hwpx" + assert result.lossiness == "low" + assert "2026ํ•™๋…„๋„ ์šด์˜ ๊ณ„ํš" in result.markdown + assert "## 1. ์ถ”์ง„ ๋ชฉ์ " in result.markdown + assert "| ํ•ญ๋ชฉ | ๋‚ด์šฉ |" in result.markdown + assert result.metadata["table_count"] == 1 + assert result.tables[0]["data"][1] == ["๊ธฐ๊ฐ„", "2026. 3."] + assert result.attempts[0].accepted is True + + +def test_document_ingestor_detects_hwpx_from_stream_without_extension(tmp_path: Path) -> None: + source = _sample_hwpx(tmp_path / "sample.hwpx") + stream = BytesIO(source.read_bytes()) + + result = DocumentIngestor.default().convert(stream) + + assert result.source_format == "hwpx" + assert "2026ํ•™๋…„๋„ ์šด์˜ ๊ณ„ํš" in result.markdown + + +def test_document_ingestor_reports_unsupported_format(tmp_path: Path) -> None: + source = tmp_path / "sample.txt" + source.write_text("plain text is a later adapter", encoding="utf-8") + + with pytest.raises(UnsupportedDocumentFormat) as exc: + DocumentIngestor.default().convert(source) + + payload = exc.value.as_dict() + assert payload["error"] == "UnsupportedDocumentFormat" + assert payload["attempts"][0]["converter"] == "HwpxMarkdownConverter" + assert payload["attempts"][0]["accepted"] is False + + +def test_normalize_markdown_trims_trailing_space_and_extra_blank_lines() -> None: + assert normalize_markdown("a \n\n\n\nb\n") == "a\n\nb"