Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ new.save_to_path("새문서.hwpx")
### 🔍 읽기 · 추출
- 텍스트/HTML/Markdown 내보내기 — `export_text()` · `export_html()` · `export_markdown()`
- **풍부한 Markdown** — `export_rich_markdown()`은 인라인 서식(`**굵게**`·`*기울임*`·`~~취소선~~`), 중첩 표(colspan/rowspan 안전), 도형 텍스트, 이미지, 각주/미주, 하이퍼링크, 제목(`#`/`##`) 자동 감지까지 보존
- **문서 ingest 게이트웨이** — `hwpx.ingest.DocumentIngestor`가 HWPX를 감지해 rich Markdown과 섹션/표 메타데이터로 정규화
- `TextExtractor` / `ObjectFinder` — 섹션·문단 순회, 태그·속성·XPath로 객체 탐색 (`hp:tab`은 `\t`로 보존, roundtrip 안전)

```python
Expand Down
18 changes: 18 additions & 0 deletions src/hwpx/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,16 @@ def __getattr__(name: str) -> object:
PreviewPage,
render_layout_preview,
)
from .ingest import (
ConversionAttempt,
DocumentConverter,
DocumentIngestError,
DocumentIngestResult,
DocumentIngestor,
DocumentSourceInfo,
HwpxMarkdownConverter,
UnsupportedDocumentFormat,
)
from .patch import (
BytePreservingPatchResult,
ParagraphTextPatch,
Expand Down Expand Up @@ -123,8 +133,14 @@ def __getattr__(name: str) -> object:
"DEFAULT_STYLE_PRESET",
"DOCUMENT_PLAN_SCHEMA_VERSION",
"get_document_plan_schema",
"ConversionAttempt",
"DocumentBlock",
"DocumentConverter",
"DocumentIngestError",
"DocumentIngestResult",
"DocumentIngestor",
"DocumentPlan",
"DocumentSourceInfo",
"DocumentStylePreset",
"EditorOpenSafetyReport",
"ParagraphInfo",
Expand All @@ -141,6 +157,7 @@ def __getattr__(name: str) -> object:
"TEMPLATE_FORMFIT_PLAN_SCHEMA_VERSION",
"TextExtractor",
"FoundElement",
"HwpxMarkdownConverter",
"ObjectFinder",
"OFFICIAL_DOCUMENT_STYLE_REPORT_VERSION",
"DOC_DIFF_REPORT_VERSION",
Expand All @@ -150,6 +167,7 @@ def __getattr__(name: str) -> object:
"STYLE_PROFILE_SCHEMA_VERSION",
"TEMPLATE_REGISTRY_SCHEMA_VERSION",
"TABLE_COMPUTE_REPORT_VERSION",
"UnsupportedDocumentFormat",
"apply_style_profile_to_plan",
"build_comparison_table_plan",
"build_image_grid",
Expand Down
26 changes: 26 additions & 0 deletions src/hwpx/ingest/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
"""Document ingest primitives for HWPX-first Markdown conversion."""

from .base import (
ConversionAttempt,
DocumentConverter,
DocumentIngestError,
DocumentIngestResult,
DocumentIngestor,
DocumentSourceInfo,
UnsupportedDocumentFormat,
normalize_markdown,
)
from .hwpx_converter import HwpxMarkdownConverter

__all__ = [
"ConversionAttempt",
"DocumentConverter",
"DocumentIngestError",
"DocumentIngestResult",
"DocumentIngestor",
"DocumentSourceInfo",
"HwpxMarkdownConverter",
"UnsupportedDocumentFormat",
"normalize_markdown",
]
230 changes: 230 additions & 0 deletions src/hwpx/ingest/base.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,230 @@
# SPDX-License-Identifier: Apache-2.0
"""Small MarkItDown-style ingest dispatcher."""

from __future__ import annotations

from dataclasses import asdict, dataclass, field
from io import BytesIO, TextIOBase
import mimetypes
from pathlib import Path
import re
from typing import Any, BinaryIO, Iterable, Protocol


@dataclass(frozen=True, kw_only=True)
class DocumentSourceInfo:
"""Metadata known or inferred about an input document."""

mimetype: str | None = None
extension: str | None = None
charset: str | None = None
filename: str | None = None
local_path: str | None = None
url: str | None = None

def copy_and_update(self, *others: "DocumentSourceInfo", **kwargs: Any) -> "DocumentSourceInfo":
data = asdict(self)
for other in others:
data.update({key: val for key, val in asdict(other).items() if val is not None})
data.update({key: val for key, val in kwargs.items() if val is not None})
return DocumentSourceInfo(**data)


@dataclass(frozen=True, kw_only=True)
class ConversionAttempt:
"""One converter decision or failure."""

converter: str
accepted: bool
error_type: str | None = None
message: str | None = None

def as_dict(self) -> dict[str, Any]:
return {key: val for key, val in asdict(self).items() if val is not None}


@dataclass(kw_only=True)
class DocumentIngestResult:
"""Normalized Markdown plus structured conversion metadata."""

markdown: str
source_info: DocumentSourceInfo
source_format: str
engine: str
engine_version: str | None = None
title: str | None = None
metadata: dict[str, Any] = field(default_factory=dict)
sections: list[dict[str, Any]] = field(default_factory=list)
tables: list[dict[str, Any]] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
attempts: list[ConversionAttempt] = field(default_factory=list)
lossiness: str = "unknown"

@property
def text_content(self) -> str:
"""Compatibility alias matching common Markdown conversion APIs."""

return self.markdown

def as_dict(self) -> dict[str, Any]:
return {
"markdown": self.markdown,
"sourceInfo": asdict(self.source_info),
"sourceFormat": self.source_format,
"engine": self.engine,
"engineVersion": self.engine_version,
"title": self.title,
"metadata": self.metadata,
"sections": self.sections,
"tables": self.tables,
"warnings": self.warnings,
"attempts": [attempt.as_dict() for attempt in self.attempts],
"lossiness": self.lossiness,
}


class DocumentIngestError(RuntimeError):
"""Base class for ingest failures with converter-attempt diagnostics."""

def __init__(self, message: str, *, attempts: Iterable[ConversionAttempt] = ()) -> None:
super().__init__(message)
self.attempts = list(attempts)

def as_dict(self) -> dict[str, Any]:
return {
"error": type(self).__name__,
"message": str(self),
"attempts": [attempt.as_dict() for attempt in self.attempts],
}


class UnsupportedDocumentFormat(DocumentIngestError):
"""Raised when no registered converter accepts the source."""


class DocumentConverter(Protocol):
"""Converter contract for a specific source family."""

name: str

def accepts(self, file_stream: BinaryIO, source_info: DocumentSourceInfo) -> bool:
"""Return True if this converter should attempt conversion."""

def convert(
self,
file_stream: BinaryIO,
source_info: DocumentSourceInfo,
**kwargs: Any,
) -> DocumentIngestResult:
"""Convert the source into normalized Markdown."""


def normalize_markdown(markdown: str) -> str:
"""Normalize whitespace in generated Markdown without changing content meaning."""

text = "\n".join(line.rstrip() for line in re.split(r"\r?\n", markdown or ""))
return re.sub(r"\n{3,}", "\n\n", text).strip()


class DocumentIngestor:
"""Priority-ordered converter dispatcher."""

def __init__(self, converters: Iterable[DocumentConverter] | None = None) -> None:
self._converters: list[tuple[float, int, DocumentConverter]] = []
self._sequence = 0
if converters is not None:
for converter in converters:
self.register_converter(converter)

@classmethod
def default(cls) -> "DocumentIngestor":
from .hwpx_converter import HwpxMarkdownConverter

ingestor = cls()
ingestor.register_converter(HwpxMarkdownConverter(), priority=0.0)
return ingestor

def register_converter(self, converter: DocumentConverter, *, priority: float = 0.0) -> None:
self._converters.append((priority, self._sequence, converter))
self._sequence += 1

def convert(
self,
source: str | Path | bytes | BinaryIO,
*,
source_info: DocumentSourceInfo | None = None,
**kwargs: Any,
) -> DocumentIngestResult:
stream, inferred = _source_to_stream(source)
info = inferred if source_info is None else inferred.copy_and_update(source_info)
attempts: list[ConversionAttempt] = []
start_pos = stream.tell()

for _priority, _seq, converter in sorted(self._converters, key=lambda item: (item[0], item[1])):
name = getattr(converter, "name", type(converter).__name__)
stream.seek(start_pos)
try:
accepted = bool(converter.accepts(stream, info))
except Exception as exc:
attempts.append(
ConversionAttempt(
converter=name,
accepted=False,
error_type=type(exc).__name__,
message=str(exc),
)
)
continue
finally:
stream.seek(start_pos)

attempts.append(ConversionAttempt(converter=name, accepted=accepted))
if not accepted:
continue

try:
result = converter.convert(stream, info, **kwargs)
except Exception as exc:
attempts[-1] = ConversionAttempt(
converter=name,
accepted=True,
error_type=type(exc).__name__,
message=str(exc),
)
stream.seek(start_pos)
continue
result.markdown = normalize_markdown(result.markdown)
result.attempts = attempts
return result

raise UnsupportedDocumentFormat("no registered converter accepted the document", attempts=attempts)


def _source_to_stream(source: str | Path | bytes | BinaryIO) -> tuple[BinaryIO, DocumentSourceInfo]:
if isinstance(source, bytes):
return BytesIO(source), DocumentSourceInfo()
if isinstance(source, Path):
return _path_to_stream(source)
if isinstance(source, str):
if source.startswith(("http:", "https:", "file:", "data:")):
raise UnsupportedDocumentFormat("URI sources are not implemented in python-hwpx ingest")
return _path_to_stream(Path(source))
if hasattr(source, "read") and callable(source.read) and not isinstance(source, TextIOBase):
stream = source
if not stream.seekable():
stream = BytesIO(stream.read())
return stream, DocumentSourceInfo()
raise TypeError(f"unsupported document source type: {type(source).__name__}")


def _path_to_stream(path: Path) -> tuple[BinaryIO, DocumentSourceInfo]:
resolved = path.expanduser().resolve()
mimetype, charset = mimetypes.guess_type(str(resolved), strict=False)
info = DocumentSourceInfo(
mimetype=mimetype,
charset=charset,
extension=resolved.suffix.lower() or None,
filename=resolved.name,
local_path=str(resolved),
)
return BytesIO(resolved.read_bytes()), info
Loading
Loading