diff --git a/docling/.agents/skills/docling/references/cli.md b/docling/.agents/skills/docling/references/cli.md index 1a1bf56688..776a759f07 100644 --- a/docling/.agents/skills/docling/references/cli.md +++ b/docling/.agents/skills/docling/references/cli.md @@ -69,7 +69,7 @@ docling scan.pdf --ocr-engine tesserocr --output /tmp/ # needs system Tesserac docling scan.pdf --ocr-engine ocrmac --output /tmp/ # macOS Vision (mac only) docling scan.pdf --force-ocr --output /tmp/ # re-OCR even extractable text docling report.pdf --no-ocr --output /tmp/ # skip OCR (faster) -docling scan.pdf --ocr-lang en --ocr-lang de --output /tmp/ # restrict languages +docling scan.pdf --ocr-lang en,de --output /tmp/ # BCP-47 tags, comma-separated ``` OCR engines are optional dependencies — see diff --git a/docling/cli/export_utils.py b/docling/cli/export_utils.py index d5a9bec7c9..e6b9717dad 100644 --- a/docling/cli/export_utils.py +++ b/docling/cli/export_utils.py @@ -83,9 +83,13 @@ def _parse_page_range(raw: str | None) -> PageRange | None: def _split_list(raw: str | None) -> list[str] | None: + """Split a comma/semicolon-separated CLI value, dropping blanks. + + Stripping matters: `--ocr-lang "en, de"` must yield `de`, not `" de"`. + """ if raw is None: return None - return re.split(r"[;,]", raw) + return [item.strip() for item in re.split(r"[;,]", raw) if item.strip()] def _is_empty_output(path: Path) -> bool: diff --git a/docling/cli/main.py b/docling/cli/main.py index b4b02c6059..14403043b6 100644 --- a/docling/cli/main.py +++ b/docling/cli/main.py @@ -11,7 +11,7 @@ from collections.abc import Iterable from enum import Enum from pathlib import Path -from typing import Annotated, Literal, Type, cast +from typing import Annotated, Any, Literal, Type, cast from urllib.parse import urlparse from docling.datamodel.service.responses import ChunkedDocumentResultItem @@ -43,7 +43,7 @@ from docling_core.transforms.visualizer.layout_visualizer import LayoutVisualizer from docling_core.types.doc import ImageRefMode from docling_core.utils.file import resolve_source_to_path -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from rich.console import Console from docling.cli.export_utils import ( @@ -161,6 +161,17 @@ from docling.models.factories.base_factory import BaseFactory from docling.utils.profiling import ProfilingItem + +def _first_error_message(err: ValidationError) -> str: + """The most useful line of a pydantic error, for a typer.BadParameter.""" + errors = err.errors() + if not errors: + return str(err) + message = errors[0].get("msg", "") + # Pydantic prefixes messages raised from a validator with "Value error, ". + return message.removeprefix("Value error, ") or str(err) + + warnings.filterwarnings(action="ignore", category=UserWarning, module="pydantic|torch") warnings.filterwarnings(action="ignore", category=FutureWarning, module="easyocr") @@ -893,7 +904,17 @@ def convert( # noqa: C901 str | None, typer.Option( ..., - help="Provide a comma-separated list of languages used by the OCR engine. Note that each OCR engine has different values for the language names.", + help=( + "Comma-separated list of OCR languages as BCP-47 tags, e.g. " + "'en,de' or 'zh-Hant'. The selected engine's own language codes " + "are accepted too and mean what that engine means by them, so " + "'--ocr-engine rapidocr --ocr-lang ch' is Simplified Chinese. " + "Omit the option for the engine's default languages, or pass an " + "empty value (--ocr-lang '') to let the engine choose, which for " + "Tesseract is per-page script detection. 'mul' selects a " + "multilingual model on the engines that ship one; to skip OCR " + "entirely use --no-ocr." + ), ), ] = None, psm: Annotated[ @@ -1234,14 +1255,21 @@ def convert( # noqa: C901 resolved_ocr_mode = OcrMode.FULL_PAGE else: resolved_ocr_mode = ocr_mode - ocr_options: OcrOptions = ocr_factory.create_options( # type: ignore - kind=ocr_engine, - mode=resolved_ocr_mode, - ) - + ocr_kwargs: dict[str, Any] = {"mode": resolved_ocr_mode} ocr_lang_list = _split_list(ocr_lang) + # `_split_list` returns None only when the option was not given, so an + # explicitly empty value reaches the engine as `lang=[]`: "your default". if ocr_lang_list is not None: - ocr_options.lang = ocr_lang_list + ocr_kwargs["lang"] = ocr_lang_list + try: + ocr_options: OcrOptions = ocr_factory.create_options( # type: ignore + kind=ocr_engine, + **ocr_kwargs, + ) + except ValidationError as err: + raise typer.BadParameter( + _first_error_message(err), param_hint="--ocr-lang" + ) from err if psm is not None and isinstance( ocr_options, TesseractOcrOptions | TesseractCliOcrOptions ): diff --git a/docling/cli/models.py b/docling/cli/models.py index 63ba346a38..a4cc1034e1 100644 --- a/docling/cli/models.py +++ b/docling/cli/models.py @@ -30,7 +30,7 @@ from docling.datamodel.settings import settings from docling.models.stages.ocr.easyocr_model import ( - _resolve_easyocr_recognition_models, + resolve_easyocr_codes, ) from docling.models.stages.ocr.rapid_ocr_model import _parse_rapidocr_model_spec from docling.models.utils.hf_model_download import download_hf_model @@ -122,7 +122,12 @@ def download( typer.Option( ..., "--easyocr-lang", - help="EasyOCR language code to prefetch. Repeat for multiple languages.", + help=( + "OCR language to prefetch for EasyOCR, as a BCP-47 tag " + "(e.g. 'de', 'zh-Hant', 'ru'). EasyOCR's own codes are accepted " + "too and mean what EasyOCR means by them, so 'ch_sim' is " + "Simplified Chinese. Repeat for multiple." + ), ), ] = None, rapidocr_backend_lang: Annotated[ @@ -132,7 +137,11 @@ def download( "--rapidocr-backend-lang", help=( "RapidOCR checkpoint set to prefetch, as ':' " - "(e.g. 'onnxruntime:el', 'torch:korean'). Repeat for multiple. Replaces the default set." + "with a BCP-47 language (e.g. 'onnxruntime:el', 'torch:ko'). " + "PP-OCR's own codes are accepted too, including its script " + "recognizers, which no language tag can name: " + "'onnxruntime:cyrillic', 'torch:ch'. Repeat for multiple. " + "Replaces the default set." ), ), ] = None, @@ -156,7 +165,7 @@ def download( param_hint="--easyocr-lang", ) try: - _resolve_easyocr_recognition_models(easyocr_lang) + resolve_easyocr_codes(easyocr_lang) except ValueError as error: raise typer.BadParameter(str(error), param_hint="--easyocr-lang") from error if rapidocr_backend_lang is not None: diff --git a/docling/cli/remote.py b/docling/cli/remote.py index 562aa4e7e6..58494c89a6 100644 --- a/docling/cli/remote.py +++ b/docling/cli/remote.py @@ -36,9 +36,33 @@ DoclingServiceClient, StatusWatcherKind, ) +from docling.utils.ocr_language import OcrLanguageResolver _log = logging.getLogger(__name__) + +def _canonicalize_ocr_lang(raw: Optional[str]) -> Optional[list[str]]: + """Canonicalize --ocr-lang locally, so a typo fails here and not remotely. + + This command has no `--ocr-engine`, so it cannot pick an engine's own + vocabulary and uses the engine-independent one: native tokens that every + engine agrees on are accepted, and the handful that clash with a BCP-47 tag + of a different language are not. + """ + # `_split_list` returns None only when the option was not given, so an + # explicitly empty value survives as `[]`: "let the engine choose". + tags = _split_list(raw) + if tags is None: + return None + try: + return [ + language.tag + for language in OcrLanguageResolver.canonicalize_ocr_languages(tags) + ] + except ValueError as err: + raise typer.BadParameter(str(err), param_hint="--ocr-lang") from err + + _REMOTE_HELP = """\ Convert documents through a remote docling-serve service instead of locally. @@ -191,7 +215,17 @@ def convert_remote( ocr_lang: Annotated[ Optional[str], typer.Option( - help="Comma-separated list of OCR languages (engine-specific names).", + help=( + "Comma-separated list of OCR languages as BCP-47 tags, e.g. " + "'en,de' or 'zh-Hant'. Widely-understood engine names such as " + "'chinese' or 'japan' are accepted too, but engine-specific ones " + "such as 'ch' are not, because this command does not choose the " + "engine. Omit the option for the service's default languages, or " + "pass an empty value (--ocr-lang '') to let the engine choose. " + "Canonicalized locally before the request is sent, so the " + "remote service must be recent enough to speak BCP-47 OCR " + "languages." + ), ), ] = None, enrich_code: Annotated[ @@ -309,7 +343,7 @@ def convert_remote( "to_formats": to_formats, "do_ocr": ocr, "force_ocr": force_ocr, - "ocr_lang": _split_list(ocr_lang), + "ocr_lang": _canonicalize_ocr_lang(ocr_lang), "do_table_structure": tables, "pipeline": pipeline, "do_code_enrichment": enrich_code, diff --git a/docling/datamodel/pipeline_options.py b/docling/datamodel/pipeline_options.py index 6c7ece51ea..a60a77c50f 100644 --- a/docling/datamodel/pipeline_options.py +++ b/docling/datamodel/pipeline_options.py @@ -75,6 +75,7 @@ ObjectDetectionEngineOptionsMixin, ) from docling.models.inference_engines.vlm.base import VlmEngineOptionsMixin +from docling.utils.ocr_language import OcrLanguageResolver _log = logging.getLogger(__name__) @@ -198,6 +199,13 @@ class OcrOptions(BaseOptions): configurations. """ + # Every concrete engine overrides this with its own discriminator + # The empty default keeps the abstract base instantiable + kind: ClassVar[str] = "" + + # Whether `lang` is canonicalized, or handed to the engine verbatim + canonicalize_lang: ClassVar[bool] = True + mode: Annotated[ OcrMode, Field( @@ -214,8 +222,18 @@ class OcrOptions(BaseOptions): lang: Annotated[ list[str], Field( - description="List of OCR languages to use. The format must match the values of the OCR engine of choice.", - examples=[["deu", "eng"]], + description=( + "OCR languages as BCP-47 tags (e.g. `en`, `de-DE`, `zh-Hant`), in " + "order of preference. Tags are canonicalized to a language-script " + "pair, so `deu`, `ger`, `de` and `de-DE` are all `de-Latn`. An empty " + "list means the engine's own default, which for Tesseract is " + "per-page script detection. One tag carries engine-independent " + "meaning and must be used alone: `mul`, the engine's broadest " + "multilingual model, on the engines that ship one. " + "A language the selected engine has no model for raises an error " + "rather than falling back silently." + ), + examples=[["de", "en"], ["zh-Hans"], []], ), ] @@ -247,6 +265,28 @@ class OcrOptions(BaseOptions): ), ] = False + model_config = ConfigDict( + validate_assignment=True, + validate_default=True, + ) + + @field_validator("lang", mode="after") + @classmethod + def _canonicalize_lang(cls, value: list[str]) -> list[str]: + """Rewrite every entry into its canonical BCP-47 form. + + Declared once on the base: pydantic collects field validators by field + name across the MRO, so it fires for the subclasses that redefine `lang` + with their own default -- unless they turn `canonicalize_lang` off, which + leaves `lang` exactly as the user wrote it. + """ + if not cls.canonicalize_lang: + return value + return [ + language.tag + for language in OcrLanguageResolver.canonicalize_ocr_languages(value) + ] + @model_validator(mode="after") def _apply_force_full_page_ocr(self) -> "OcrOptions": r""" @@ -256,7 +296,9 @@ def _apply_force_full_page_ocr(self) -> "OcrOptions": with warnings.catch_warnings(): # deprecated force_full_page_ocr warnings.filterwarnings("ignore", category=DeprecationWarning) forced = self.force_full_page_ocr - if forced: + # `validate_assignment` re-runs this validator on every assignment, so + # the write must be skipped once `mode` already holds the forced value. + if forced and self.mode is not OcrMode.FULL_PAGE: self.mode = OcrMode.FULL_PAGE return self @@ -266,13 +308,16 @@ class OcrAutoOptions(OcrOptions): When this option is used, Docling probes the runtime environment at pipeline initialization and selects the best available OCR engine - (e.g., EasyOCR if GPU is present, Tesseract otherwise). Language - settings are deferred to the chosen engine's defaults. + (e.g., EasyOCR if GPU is present, Tesseract otherwise). The requested + languages are forwarded to whichever engine is chosen, and an engine with + no model for them is skipped in favour of the next candidate. Notes: - The `lang` field is intentionally defaulted to an empty list. - To control language selection, specify an explicit OCR engine - option class instead. + `lang` is forwarded to whichever engine is selected. The default empty + list means "each engine's own default model", so leaving it alone + reproduces the behaviour of picking that engine by hand. An engine that + cannot serve the requested language is treated as unavailable and the + next candidate is probed. """ kind: ClassVar[Literal["auto"]] = "auto" @@ -280,9 +325,12 @@ class OcrAutoOptions(OcrOptions): list[str], Field( description=( - "The automatic OCR engine will use the default values of the engine. Please specify the engine " - "explicitly to change the language selection." - ) + "OCR languages as BCP-47 tags, forwarded to the automatically " + "selected engine. The default empty list leaves the choice of model " + "to that engine. Engines with no model for the requested language " + "are skipped during selection." + ), + examples=[[], ["de", "en"]], ), ] = [] @@ -300,18 +348,21 @@ class RapidOcrOptions(OcrOptions): list[str], Field( description=( - "Recognition language. RapidOCR uses a single language per run; if more than one " - "value is given only the first is used. Accepted values resolve to a PP-OCR " - "recognizer: PP-OCRv6 covers ~52 language codes (e.g. 'ch', 'en', 'de', 'fr', " - "'japan'; the docling defaults 'chinese'/'english' map to 'ch'/'en'). Script-family " - "names route to PP-OCRv5 on the onnxruntime/openvino/paddle backends ('arabic', " - "'ch', 'cyrillic', 'devanagari', 'el', 'en', 'eslav', 'korean', 'latin', 'ta', " - "'te', 'th') or to PP-OCRv4 on the torch backend ('arabic', 'cyrillic', " - "'devanagari', 'ka', 'korean', 'latin', 'ta', 'te'). A language the resolved " - "backend cannot serve raises an error rather than falling back silently." - ) + "Recognition language as a BCP-47 tag. RapidOCR runs a single " + "language per run; if more than one tag is given the first is used " + "and the rest are ignored with a warning. Tags are mapped onto a " + "PP-OCR recognizer: PP-OCRv6 covers ~52 languages, and a language " + "PP-OCR serves only through a script-wide recognizer routes to " + "PP-OCRv5 (onnxruntime/openvino/paddle) or PP-OCRv4 (torch). " + "PP-OCR's script recognizers can also be named directly with their " + "own tokens (`latin`, `cyrillic`, `arabic`, `devanagari`). An empty " + "list selects the Simplified Chinese default; `mul` is not " + "supported. A language the resolved backend cannot serve raises an " + "error rather than falling back silently." + ), + examples=[["zh-Hans"], ["en"], ["cyrillic"]], ), - ] = ["chinese"] + ] = ["zh-Hans"] backend: Annotated[ Literal["onnxruntime", "openvino", "paddle", "torch"], Field( @@ -419,8 +470,14 @@ class NemotronOcrOptions(OcrOptions): list[str], Field( description=( - "List of OCR languages. nemotron-OCR-v2 supports 'english' and 'multilingual'" - ) + "Recognition language as a BCP-47 tag. nemotron-OCR-v2 ships two " + "recognizers: `en` and an empty list both select the English " + "model, while `mul` and the languages the multilingual model " + "covers (`zh-Hans`, `zh-Hant`, `ja`, `ko`, `ru`) select the " + "multilingual one. Any other language raises; use `mul` to opt " + "into the multilingual model explicitly." + ), + examples=[["en"], ["mul"]], ), ] = [] merge_level: Annotated[ @@ -454,11 +511,16 @@ class EasyOcrOptions(OcrOptions): list[str], Field( description=( - "List of language codes for OCR. EasyOCR supports 80+ languages. Use ISO 639-1 codes " - "(e.g., `en`, `fr`, `de`). Multiple languages can be specified for multilingual documents." - ) + "OCR languages as BCP-47 tags. EasyOCR covers 80+ languages and " + "runs several at once, but they must share a recognition model, so " + "keep the list short and script-consistent. Each language is routed " + "to the recognition network of its script, so `ru` reaches the " + "Cyrillic model. `mul` is not supported -- list the languages " + "explicitly." + ), + examples=[["fr", "de", "es", "en"], ["ru", "uk"]], ), - ] = ["fr", "de", "es", "en"] + ] = ["en-Latn", "es-Latn", "fr-Latn", "de-Latn"] use_gpu: Annotated[ bool | None, Field( @@ -527,11 +589,18 @@ class TesseractCliOcrOptions(OcrOptions): list[str], Field( description=( - "List of Tesseract language codes. Use 3-letter ISO 639-2 codes (e.g., `eng`, `fra`, `deu`). " - "Multiple languages enable multilingual OCR. Requires corresponding Tesseract language data files." - ) + "OCR languages as BCP-47 tags, mapped onto the installed tessdata " + "files (`de` becomes `deu`, `zh-Hant` becomes `chi_tra`). Multiple " + "languages enable multilingual OCR and are joined in the order " + "given, which Tesseract treats as preference order. A `script/` " + "traineddata file can be named directly (`script/Cyrillic`), and an " + "empty list runs orientation and script detection per page " + "(requires the `osd` traineddata). Languages without an installed " + "traineddata file raise at construction time." + ), + examples=[["fr", "de", "es", "en"], []], ), - ] = ["fra", "deu", "spa", "eng"] + ] = ["fr-Latn", "de-Latn", "es-Latn", "en-Latn"] tesseract_cmd: Annotated[ str, Field( @@ -572,11 +641,18 @@ class TesseractOcrOptions(OcrOptions): list[str], Field( description=( - "List of Tesseract language codes. Use 3-letter ISO 639-2 codes (e.g., `eng`, `fra`, `deu`). " - "Multiple languages enable multilingual OCR. Requires corresponding Tesseract language data files." - ) + "OCR languages as BCP-47 tags, mapped onto the installed tessdata " + "files (`de` becomes `deu`, `zh-Hant` becomes `chi_tra`). Multiple " + "languages enable multilingual OCR and are joined in the order " + "given, which Tesseract treats as preference order. A `script/` " + "traineddata file can be named directly (`script/Cyrillic`), and an " + "empty list runs orientation and script detection per page " + "(requires the `osd` traineddata). Languages without an installed " + "traineddata file raise at construction time." + ), + examples=[["fr", "de", "es", "en"], []], ), - ] = ["fra", "deu", "spa", "eng"] + ] = ["fr-Latn", "de-Latn", "es-Latn", "en-Latn"] path: Annotated[ str | None, Field( @@ -608,11 +684,14 @@ class OcrMacOptions(OcrOptions): list[str], Field( description=( - "List of language locale codes for macOS OCR. Use format `language-REGION` (e.g., `en-US`, `fr-FR`). " - "Leverages native macOS Vision framework for OCR on Apple platforms." - ) + "OCR languages as BCP-47 tags, matched against the recognition " + "languages the running macOS reports (`de` becomes `de-DE`, `pt` " + "becomes `pt-BR`). An empty list hands the choice to Vision's own " + "automatic behaviour. `mul` is not supported." + ), + examples=[["fr", "de", "es", "en"], []], ), - ] = ["fr-FR", "de-DE", "es-ES", "en-US"] + ] = ["fr-Latn", "de-Latn", "es-Latn", "en-Latn"] recognition: Annotated[ str, Field( @@ -654,6 +733,10 @@ class KserveV2OcrOptions(OcrOptions, KserveV2OptionsMixin): kind: ClassVar[Literal["kserve_v2_ocr"]] = "kserve_v2_ocr" + # The deployed model is the only authority on the languages it serves, and + # docling cannot inspect it, so `lang` is neither validated nor mapped here. + canonicalize_lang: ClassVar[bool] = False + model_name: str = Field( default="ocr", description="Remote model name registered in the KServe v2 endpoint.", @@ -664,8 +747,8 @@ class KserveV2OcrOptions(OcrOptions, KserveV2OptionsMixin): Field( description=( "List of OCR languages. Note: Language selection depends on the deployed model. " - "This parameter is passed to the server but may not be used by all models." - ) + ), + examples=[["english"], ["chinese"]], ), ] = ["english", "chinese"] diff --git a/docling/datamodel/service/options.py b/docling/datamodel/service/options.py index 9a45170dd2..b95658df76 100644 --- a/docling/datamodel/service/options.py +++ b/docling/datamodel/service/options.py @@ -378,12 +378,13 @@ class ConvertDocumentsOptions(BaseModel): Optional[list[str]], Field( description=( - "List of languages used by the OCR engine. " - "Note that each OCR engine has " - "different values for the language names. String or list of strings. " - "Optional, defaults to empty." + "OCR languages as BCP-47 tags (e.g. `en`, `de-DE`, `zh-Hant`), in " + "order of preference. The service canonicalizes them to a " + "language-script pair, so `deu`, `ger` and `de-DE` are all German. " + "The reserved tag `mul` must be used alone. Optional; " + "the selected engine's default applies when omitted or empty." ), - examples=[["fr", "de", "es", "en"]], + examples=[["fr", "de", "es", "en"], ["zh-Hant"], []], ), ] = None diff --git a/docling/exceptions.py b/docling/exceptions.py index 55073fc3a4..8e6eff0ac3 100644 --- a/docling/exceptions.py +++ b/docling/exceptions.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: The Docling Contributors # SPDX-License-Identifier: MIT +from collections.abc import Sequence + class BaseError(RuntimeError): pass @@ -29,3 +31,30 @@ class SecurityError(BaseError): class AcceleratorDeviceNotAvailableError(BaseError): """Raised when an explicitly requested accelerator device is not available.""" + + +class OcrLanguageNotSupportedError(BaseError): + """Raised when an OCR engine has no model for a requested language. + + Docling never silently substitutes a different recognizer: when the + canonicalized request cannot be served, the engine says so and names what it + does support. + """ + + def __init__( + self, + engine: str, + language: str, + supported: "Sequence[str] | None" = None, + detail: str | None = None, + ): + self.engine = engine + self.language = language + self.detail = detail + self.supported = list(supported) if supported is not None else [] + message = f"{engine} has no model for the OCR language {language!r}." + if detail: + message = f"{message} {detail}" + if self.supported: + message = f"{message} Supported: {', '.join(self.supported)}." + super().__init__(message) diff --git a/docling/models/base_ocr_model.py b/docling/models/base_ocr_model.py index e718a06feb..8639730234 100644 --- a/docling/models/base_ocr_model.py +++ b/docling/models/base_ocr_model.py @@ -7,6 +7,7 @@ from collections.abc import Iterable from enum import Enum from pathlib import Path +from typing import ClassVar import numpy as np from docling_core.types.doc import BoundingBox, CoordOrigin, Size @@ -27,7 +28,13 @@ from docling.datamodel.pipeline_options import OcrMode, OcrOptions from docling.datamodel.settings import settings from docling.datamodel.spatial import BoundingBoxSpatialIndex +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_model import BaseModelWithOptions, BasePageModel +from docling.utils.ocr_language import ( + OcrLanguage, + OcrLanguageResolver, + OcrLanguageSupport, +) _log = logging.getLogger(__name__) @@ -130,6 +137,10 @@ class BaseOcrModel(BasePageModel, BaseModelWithOptions): DEFAULT_DILATION_SIZE = 20 + #: What this engine can do with a language request. Engines override it; + #: the conservative default suits a single-model, single-language engine. + language_support: ClassVar[OcrLanguageSupport] = OcrLanguageSupport() + def __init__( self, *, @@ -141,6 +152,74 @@ def __init__( self.enabled = enabled self.options = options + # Translate options.lang into a list of OcrLanguage + self.languages: list[OcrLanguage] = ( + OcrLanguageResolver.canonicalize_ocr_languages(options.lang) + if options.canonicalize_lang + else [] + ) + + @property + def _engine_name(self) -> str: + """Human-readable engine name for coverage errors.""" + return type(self).__name__.removesuffix("Model") + + def supported_ocr_languages(self) -> list[str]: + """Canonical tags this *instance* can serve, for error messages. + + May be runtime-derived: the installed tessdata files, the selected + RapidOCR backend, the macOS version. An empty list means "unknown". + """ + return [] + + def map_ocr_language(self, language: OcrLanguage) -> str | list[str]: + """Map one canonical tag onto this engine's native code(s). + + A list covers an engine that answers one request with several codes; + most engines return a single code. + + Raises: + OcrLanguageNotSupportedError: The engine has no model for it. + """ + if language.is_passthrough or language.is_multilingual: + # A passthrough names a script recognizer of *some* engine; an engine + # that has not overridden this method does not have one. + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + detail="This engine needs an explicit language.", + ) + # Most ISO-639 engines want the primary subtag and nothing else. + return language.bcp47_language + + def resolve_ocr_languages(self) -> list[str]: + """Turn the canonical request into the native codes to hand the engine. + + An empty request stays empty: `lang=[]` means "the engine's own default", + and each engine decides what that is when it reads the result. + + Applies the two uniform policies: too many languages for the engine are + dropped with a warning (list order is preference order), and a language + with no model is an error, never a silent substitution. + """ + languages = list(self.languages) + if not self.language_support.multiple_languages and len(languages) > 1: + _log.warning( + "%s handles one OCR language at a time. Using %s and ignoring %s; " + "the order of `lang` is the order of preference.", + self._engine_name, + [languages[0].tag], + [lang.tag for lang in languages[1:]], + ) + languages = languages[:1] + + codes: list[str] = [] + for language in languages: + mapped = self.map_ocr_language(language) + codes.extend([mapped] if isinstance(mapped, str) else mapped) + return list(dict.fromkeys(codes)) + def get_ocr_rects(self, page: Page) -> list[BoundingBox]: r""" Produce the input rects for the OCR according to the logic for each OcrMode diff --git a/docling/models/stages/ocr/auto_ocr_model.py b/docling/models/stages/ocr/auto_ocr_model.py index 976f816409..3ff1151f5c 100644 --- a/docling/models/stages/ocr/auto_ocr_model.py +++ b/docling/models/stages/ocr/auto_ocr_model.py @@ -5,7 +5,7 @@ import sys from collections.abc import Iterable from pathlib import Path -from typing import Optional, Type +from typing import NamedTuple, Optional, Type from docling.datamodel.accelerator_options import AcceleratorOptions from docling.datamodel.base_models import Page @@ -18,6 +18,7 @@ OcrOptions, RapidOcrOptions, ) +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_ocr_model import BaseOcrModel from docling.models.stages.ocr.easyocr_model import EasyOcrModel from docling.models.stages.ocr.nemotron_ocr_model import NemotronOcrModel @@ -26,6 +27,22 @@ _log = logging.getLogger(__name__) +_NOT_INSTALLED = "not installed" + + +class _Rejection(NamedTuple): + """Why one candidate engine was passed over. + + `is_language` separates "this engine has no model for the requested + language" from every other reason -- not installed, no CUDA, wrong + platform -- because only the former makes the aggregated failure a + language problem. + """ + + engine: str + reason: str + is_language: bool + class OcrAutoModel(BaseOcrModel): def __init__( @@ -44,6 +61,8 @@ def __init__( self.options: OcrAutoOptions self._engine: Optional[BaseOcrModel] = None + # Why each candidate was passed over, for the aggregated error below. + rejected: list[_Rejection] = [] if self.enabled: if "darwin" == sys.platform: try: @@ -54,12 +73,19 @@ def __init__( artifacts_path=artifacts_path, options=OcrMacOptions( mode=self.options.mode, + lang=self.options.lang, ), accelerator_options=accelerator_options, ) _log.info("Auto OCR model selected ocrmac.") except ImportError: _log.info("ocrmac cannot be used because ocrmac is not installed.") + rejected.append( + _Rejection("ocrmac", _NOT_INSTALLED, is_language=False) + ) + except OcrLanguageNotSupportedError as exc: + _log.info("Auto OCR: skipping ocrmac: %s", exc) + rejected.append(_Rejection("ocrmac", str(exc), is_language=True)) if "linux" == sys.platform: try: @@ -72,14 +98,25 @@ def __init__( artifacts_path=artifacts_path, options=NemotronOcrOptions( mode=self.options.mode, + lang=self.options.lang, ), accelerator_options=accelerator_options, ) _log.info("Auto OCR model selected nemotron.") except ImportError: _log.info("Nemotron cannot be used because it is not installed.") + rejected.append( + _Rejection("nemotron", _NOT_INSTALLED, is_language=False) + ) + except OcrLanguageNotSupportedError as exc: + # Caught before the arm below: OcrLanguageNotSupportedError + # is a BaseError, hence a RuntimeError, and the two must not + # be reported as the same kind of failure. + _log.info("Auto OCR: skipping nemotron: %s", exc) + rejected.append(_Rejection("nemotron", str(exc), is_language=True)) except (RuntimeError, FileNotFoundError) as exc: _log.warning("Nemotron OCR cannot be used: %s", exc) + rejected.append(_Rejection("nemotron", str(exc), is_language=False)) if self._engine is None: try: @@ -92,6 +129,7 @@ def __init__( options=RapidOcrOptions( backend="onnxruntime", mode=self.options.mode, + lang=self.options.lang, ), accelerator_options=accelerator_options, ) @@ -100,6 +138,16 @@ def __init__( _log.info( "rapidocr cannot be used because onnxruntime is not installed." ) + rejected.append( + _Rejection( + "rapidocr (onnxruntime)", _NOT_INSTALLED, is_language=False + ) + ) + except OcrLanguageNotSupportedError as exc: + _log.info("Auto OCR: skipping rapidocr (onnxruntime): %s", exc) + rejected.append( + _Rejection("rapidocr (onnxruntime)", str(exc), is_language=True) + ) if self._engine is None: try: @@ -110,12 +158,19 @@ def __init__( artifacts_path=artifacts_path, options=EasyOcrOptions( mode=self.options.mode, + lang=self.options.lang, ), accelerator_options=accelerator_options, ) _log.info("Auto OCR model selected easyocr.") except ImportError: _log.info("easyocr cannot be used because it is not installed.") + rejected.append( + _Rejection("easyocr", _NOT_INSTALLED, is_language=False) + ) + except OcrLanguageNotSupportedError as exc: + _log.info("Auto OCR: skipping easyocr: %s", exc) + rejected.append(_Rejection("easyocr", str(exc), is_language=True)) if self._engine is None: try: @@ -128,6 +183,7 @@ def __init__( options=RapidOcrOptions( backend="torch", mode=self.options.mode, + lang=self.options.lang, ), accelerator_options=accelerator_options, ) @@ -136,8 +192,28 @@ def __init__( _log.info( "rapidocr cannot be used because rapidocr or torch is not installed." ) + rejected.append( + _Rejection( + "rapidocr (torch)", _NOT_INSTALLED, is_language=False + ) + ) + except OcrLanguageNotSupportedError as exc: + _log.info("Auto OCR: skipping rapidocr (torch): %s", exc) + rejected.append( + _Rejection("rapidocr (torch)", str(exc), is_language=True) + ) if self._engine is None: + if any(rejection.is_language for rejection in rejected): + listed = "\n".join( + f" - {rejection.engine}: {rejection.reason}" + for rejection in rejected + ) + raise OcrLanguageNotSupportedError( + "Automatic OCR engine selection", + ", ".join(self.options.lang), + detail=f"No installed engine can serve it:\n{listed}", + ) _log.warning("No OCR engine found. Please review the install details.") def __call__( diff --git a/docling/models/stages/ocr/easyocr_model.py b/docling/models/stages/ocr/easyocr_model.py index f85860a068..6a0fc315b9 100644 --- a/docling/models/stages/ocr/easyocr_model.py +++ b/docling/models/stages/ocr/easyocr_model.py @@ -6,6 +6,7 @@ import warnings import zipfile from collections.abc import Iterable +from functools import lru_cache from pathlib import Path from typing import List, Optional, Type @@ -21,16 +22,57 @@ OcrOptions, ) from docling.datamodel.settings import settings -from docling.exceptions import SecurityError +from docling.exceptions import OcrLanguageNotSupportedError, SecurityError from docling.models.base_ocr_model import BaseOcrModel from docling.utils.accelerator_utils import decide_device +from docling.utils.ocr_language import ( + OcrLanguage, + OcrLanguageResolver, + OcrLanguageSupport, +) from docling.utils.profiling import TimeRecorder from docling.utils.utils import download_url_with_progress _log = logging.getLogger(__name__) -def _resolve_easyocr_recognition_models(languages: Iterable[str]) -> List[str]: +# Canonical tag -> EasyOCR code, where EasyOCR deviates from ISO 639-1. +_EASYOCR_CODES: dict[str, str] = { + "zh-Hans": "ch_sim", + "zh-Hant": "ch_tra", + "sr-Cyrl": "rs_cyrillic", + "sr-Latn": "rs_latin", + "tg-Cyrl": "tjk", + "fil-Latn": "tl", + # EasyOCR names these two with their ISO 639-3 codes; canonicalization + # reaches the 639-1 `av`/`ce`, which EasyOCR has no recognizer under. + "av-Cyrl": "ava", + "ce-Cyrl": "che", + # EasyOCR's `ang` is Angika and its `mah` is Magahi, both Devanagari. CLDR + # gives Angika a likely script of Latin and normalizes `mah` to Marshallese, + # so neither is reachable without an explicit entry. + "anp-Deva": "ang", + "mag-Deva": "mah", + # Tabasaran is written in Cyrillic; CLDR's likely script for it is Latin. + "tab-Cyrl": "tab", +} + +_EASYOCR_CODE_TO_TAG: dict[str, str] = { + code: tag for tag, code in _EASYOCR_CODES.items() +} + +# EasyOCR has no "engine default": `lang_list` is a required positional argument, and +# an empty one leaves the reader with a symbols-only character set. Name one language. +_EASYOCR_DEFAULT_LANGUAGE = "en" + + +@lru_cache(maxsize=1) +def _easyocr_code_to_model() -> dict[str, str]: + """EasyOCR code -> the recognition checkpoint that serves it. + + Doubles as EasyOCR's supported-language vocabulary: a code absent from this + mapping has no recognizer. + """ from easyocr.config import ( arabic_lang_list, bengali_lang_list, @@ -40,6 +82,8 @@ def _resolve_easyocr_recognition_models(languages: Iterable[str]) -> List[str]: ) language_models: dict[str, str] = {} + + # First add the languages that come from big language groups for language_group, model_name in ( (latin_lang_list, "latin_g2"), (arabic_lang_list, "arabic_g1"), @@ -48,6 +92,8 @@ def _resolve_easyocr_recognition_models(languages: Iterable[str]) -> List[str]: (devanagari_lang_list, "devanagari_g1"), ): language_models.update(dict.fromkeys(language_group, model_name)) + + # Add other supported languages, which are outside of the lang_lists. Overwrite "en". language_models.update( { "en": "english_g2", @@ -61,21 +107,65 @@ def _resolve_easyocr_recognition_models(languages: Iterable[str]) -> List[str]: "kn": "kannada_g2", } ) - - model_names: List[str] = [] - for language in languages: + return language_models + + +def _easyocr_code(language: OcrLanguage) -> Optional[str]: + """The EasyOCR code for a canonical language, or `None` when there is no model.""" + if language.is_passthrough: + # `ch_sim`, `ang`: one of EasyOCR's own codes, handed over as written. + code = language.native + elif language.is_multilingual: + # EasyOCR's codes are all language codes: it has no multilingual model. + return None + else: + code = _EASYOCR_CODES.get(language.bcp47) + if code is None: + # EasyOCR's codes are language-based, so the primary subtag only + # identifies the right model when the script is the usual one: its `az` + # is Latin Azerbaijani, not `az-Cyrl`. + if not language.has_default_script: + return None + code = language.bcp47_language + return code if code in _easyocr_code_to_model() else None + + +def resolve_easyocr_codes(tags: Iterable[str]) -> List[str]: + """Canonicalize language tags into the EasyOCR codes they name. + + Accepts EasyOCR's own codes as well as BCP-47, matching what + `EasyOcrOptions.lang` accepts. Used by the prefetcher, which has no model + instance to ask. + """ + codes: List[str] = [] + for tag in tags: + language = OcrLanguageResolver.canonicalize_ocr_language(tag) + code = _easyocr_code(language) + if code is None: + raise ValueError(f"Unsupported EasyOCR language: {tag}") + if code not in codes: + codes.append(code) + return codes + + +def _resolve_easyocr_recognition_models(codes: Iterable[str]) -> List[str]: + """Map EasyOCR codes onto the checkpoints the prefetcher must fetch.""" + code_to_model = _easyocr_code_to_model() + + model_names: set[str] = set() + for code in codes: try: - model_name = language_models[language] + model_names.add(code_to_model[code]) except KeyError: - raise ValueError(f"Unsupported EasyOCR language code: {language}") from None - if model_name not in model_names: - model_names.append(model_name) - return model_names + raise ValueError(f"Unsupported EasyOCR language code: {code}") from None + return sorted(model_names) class EasyOcrModel(BaseOcrModel): _model_repo_folder = "EasyOcr" + language_support = OcrLanguageSupport(multiple_languages=True) + def __init__( self, enabled: bool, @@ -93,6 +183,7 @@ def __init__( # multiplier for 72 dpi; the default 3.0 == 216 dpi. self.scale = self.options.scale + self._native_codes: List[str] = [] if self.enabled: try: @@ -103,6 +194,12 @@ def __init__( "Alternatively, Docling has support for other OCR engines. See the documentation." ) + self._native_codes = ( + self.resolve_ocr_languages() + if self.languages + else [_EASYOCR_DEFAULT_LANGUAGE] + ) + if self.options.use_gpu is None: device = decide_device(accelerator_options.device) # Enable easyocr GPU if running on CUDA, MPS @@ -131,7 +228,7 @@ def __init__( if self.options.suppress_mps_warnings: warnings.filterwarnings("ignore", message=".*pin_memory.*MPS.*") self.reader = easyocr.Reader( - lang_list=self.options.lang, + lang_list=self._native_codes, gpu=use_gpu, model_storage_directory=model_storage_directory, recog_network=self.options.recog_network, @@ -139,6 +236,27 @@ def __init__( verbose=False, ) + def supported_ocr_languages(self) -> List[str]: + tags = set() + for code in _easyocr_code_to_model(): + tag = _easyocr_code_to_tag(code) + if tag is not None: + tags.add(tag) + return sorted(tags) + + def map_ocr_language(self, language: OcrLanguage) -> str | List[str]: + code = _easyocr_code(language) + if code is None: + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + detail="EasyOCR has no multilingual model; list the languages explicitly." + if language.is_multilingual + else None, + ) + return code + @staticmethod def download_models( detection_models: List[str] = ["craft"], @@ -258,3 +376,13 @@ def __call__( @classmethod def get_options_type(cls) -> Type[OcrOptions]: return EasyOcrOptions + + +def _easyocr_code_to_tag(code: str) -> Optional[str]: + """Render one EasyOCR language code back as a canonical tag.""" + if code in _EASYOCR_CODE_TO_TAG: + return _EASYOCR_CODE_TO_TAG[code] + language = OcrLanguageResolver.canonicalize_ocr_language( + code, raise_exception=False + ) + return None if language is None else language.tag diff --git a/docling/models/stages/ocr/kserve_v2_ocr_model.py b/docling/models/stages/ocr/kserve_v2_ocr_model.py index db3886bb49..688e045c30 100644 --- a/docling/models/stages/ocr/kserve_v2_ocr_model.py +++ b/docling/models/stages/ocr/kserve_v2_ocr_model.py @@ -61,6 +61,7 @@ def __init__( artifacts_path: Path to model artifacts (not used for remote inference). options: KServe v2 OCR configuration options. accelerator_options: Accelerator configuration (not used for remote inference). + default_language: Language sent when `options.lang` is empty. """ super().__init__( enabled=enabled, @@ -74,6 +75,15 @@ def __init__( if self.enabled: self._initialize_client() + # Keep only the first language and warn + if len(options.lang) > 1: + _log.warning( + "KServe v2 OCR sends one language at a time. Using %r and " + "ignoring %s; the order of `lang` is the order of preference.", + options.lang[0], + options.lang[1:], + ) + # Prepare the lang_input during the initialization as it stays the same for all requests self._lang = options.lang[0] if len(options.lang) > 0 else default_language self._lang_input = np.array([[self._lang]], dtype=object) diff --git a/docling/models/stages/ocr/nemotron_ocr_model.py b/docling/models/stages/ocr/nemotron_ocr_model.py index 74d1817304..48c7c5fa28 100644 --- a/docling/models/stages/ocr/nemotron_ocr_model.py +++ b/docling/models/stages/ocr/nemotron_ocr_model.py @@ -23,9 +23,11 @@ OcrOptions, ) from docling.datamodel.settings import settings +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_ocr_model import BaseOcrModel from docling.models.utils.hf_model_download import download_hf_model from docling.utils.accelerator_utils import decide_device +from docling.utils.ocr_language import OcrLanguage, OcrLanguageSupport from docling.utils.profiling import TimeIntervalRecorder _log = logging.getLogger(__name__) @@ -36,10 +38,14 @@ _NEMOTRON_OCR_ENGLISH = "english" _NEMOTRON_OCR_MULTILINGUAL = "multilingual" -_NEMOTRON_OCR_ENGLISH_GROUP = ["en", "eng", "english"] + +# Canonical tags the multilingual recognizer is trained on. +_NEMOTRON_OCR_MULTILINGUAL_TAGS = frozenset( + {"zh-Hans", "zh-Hant", "ja-Jpan", "ko-Kore", "ru-Cyrl"} +) # Mappings of nemotron language to the artifacts subdir -_NEMOTRON_OCR_LANG_TO_ARTIFACT_PATHS = { +_NEMOTRON_CODE_TO_ARTIFACT = { _NEMOTRON_OCR_ENGLISH: "v2_english", _NEMOTRON_OCR_MULTILINGUAL: "v2_multilingual", } @@ -49,25 +55,6 @@ def nemotron_ocr_model_dir() -> str: return _NEMOTRON_OCR_REPO_ID.replace("/", "--") -def resolve_nemotronocr_language(req_languages: list[str] | None) -> str: - r""" - Map requested languages onto the nemotron-ocr language info - """ - if not req_languages: - # Use english by default - return _NEMOTRON_OCR_ENGLISH - - # Map request language to nemotron language - for language in req_languages: - # "en-US" / "en_US" -> "en" - normalized = language.strip().lower().replace("_", "-").split("-")[0] - - # Use the multilingual model to cover english and any non-english language - if normalized not in _NEMOTRON_OCR_ENGLISH_GROUP: - return _NEMOTRON_OCR_MULTILINGUAL - return _NEMOTRON_OCR_ENGLISH - - class NemotronOcrPrediction(TypedDict): """Exact prediction schema returned by `nemotron_ocr`.""" @@ -109,6 +96,8 @@ class _BufferedRect: class NemotronOcrModel(BaseOcrModel): r"""Wrapper for Nvidia's nemotron-ocr-v2 model""" + language_support = OcrLanguageSupport(multiple_languages=False) + def __init__( self, enabled: bool, @@ -141,17 +130,45 @@ def __init__( "Python 3.12 and CUDA 13.x." ) from exc - # Resolve the request language - language = resolve_nemotronocr_language(options.lang) + # Resolve the request language. An empty `lang` list means "the + # engine's own default", which for nemotron-OCR is English. + codes = self.resolve_ocr_languages() + code = codes[0] if codes else _NEMOTRON_OCR_ENGLISH # Initialize the model - model_dir = self._resolve_model_dir(language, artifacts_path=artifacts_path) + model_dir = self._resolve_model_dir(code, artifacts_path=artifacts_path) self.reader = NemotronOCRV2( model_dir=None if model_dir is None else str(model_dir), - lang=language, + lang=code, ) + def supported_ocr_languages(self) -> list[str]: + return ["en-Latn", "mul", *sorted(_NEMOTRON_OCR_MULTILINGUAL_TAGS)] + + def map_ocr_language(self, language: OcrLanguage) -> str: + if language.is_passthrough: + # `english`, `multilingual`: nemotron's own recognizer names. + if language.native in _NEMOTRON_CODE_TO_ARTIFACT: + assert language.native is not None + return language.native + elif language.is_multilingual: + return _NEMOTRON_OCR_MULTILINGUAL + else: + if language.bcp47 == "en-Latn": + return _NEMOTRON_OCR_ENGLISH + if language.bcp47 in _NEMOTRON_OCR_MULTILINGUAL_TAGS: + return _NEMOTRON_OCR_MULTILINGUAL + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + detail=( + "nemotron-OCR-v2 ships an English and a multilingual recognizer " + "only; use 'mul' to run the multilingual one." + ), + ) + @staticmethod def _fail_runtime(message: str) -> None: _log.error(message) @@ -192,15 +209,13 @@ def validate_runtime(cls, accelerator_options: AcceleratorOptions) -> None: ) def _resolve_model_dir( - self, language: str, artifacts_path: Optional[Path] + self, code: str, artifacts_path: Optional[Path] ) -> Optional[Path]: if artifacts_path is None: return None nemotron_lang_dir = ( - artifacts_path - / nemotron_ocr_model_dir() - / _NEMOTRON_OCR_LANG_TO_ARTIFACT_PATHS[language] + artifacts_path / nemotron_ocr_model_dir() / _NEMOTRON_CODE_TO_ARTIFACT[code] ) if nemotron_lang_dir.is_dir() and all( (nemotron_lang_dir / f).is_file() for f in self._nemotron_checkpoint_files diff --git a/docling/models/stages/ocr/ocr_mac_model.py b/docling/models/stages/ocr/ocr_mac_model.py index 1701e94cbb..5f308dea40 100644 --- a/docling/models/stages/ocr/ocr_mac_model.py +++ b/docling/models/stages/ocr/ocr_mac_model.py @@ -19,13 +19,55 @@ OcrOptions, ) from docling.datamodel.settings import settings +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_ocr_model import BaseOcrModel +from docling.utils.ocr_language import ( + OcrLanguage, + OcrLanguageResolver, + OcrLanguageSupport, +) from docling.utils.profiling import TimeRecorder _log = logging.getLogger(__name__) +# Recognition languages of a recent macOS, used when Vision cannot be queried. +# The real list is OS-version dependent, so it is only a fallback. +_OCRMAC_FALLBACK_LANGUAGES: tuple[str, ...] = ( + "en-US", + "fr-FR", + "it-IT", + "de-DE", + "es-ES", + "pt-BR", + "zh-Hans", + "zh-Hant", + "ko-KR", + "ja-JP", + "ru-RU", + "uk-UA", + "th-TH", + "vi-VT", +) + + +def _vision_vocabulary() -> list[str]: + """The recognition languages the running macOS reports, or the fallback.""" + try: + import Vision + + # pyobjc exposes the ObjC classes dynamically, so ty cannot see them. + request = Vision.VNRecognizeTextRequest.alloc().init() # ty: ignore[unresolved-attribute] + languages, error = request.supportedRecognitionLanguagesAndReturnError_(None) + if error is None and languages: + return [str(language) for language in languages] + except Exception as exc: # pyobjc/Vision availability varies by OS version + _log.debug("Could not query Vision for recognition languages: %s", exc) + return list(_OCRMAC_FALLBACK_LANGUAGES) + class OcrMacModel(BaseOcrModel): + language_support = OcrLanguageSupport(multiple_languages=True) + def __init__( self, enabled: bool, @@ -43,6 +85,8 @@ def __init__( # multiplier for 72 dpi; the default 3.0 == 216 dpi. self.scale = self.options.scale + self._native_codes: list[str] = [] + self._vision_vocabulary: list[str] = [] if self.enabled: if "darwin" != sys.platform: @@ -60,6 +104,53 @@ def __init__( self.reader_RIL = ocrmac.OCR + self._vision_vocabulary = _vision_vocabulary() + self._native_codes = self.resolve_ocr_languages() + + def supported_ocr_languages(self) -> list[str]: + # Map the Vision language tags to the canonical tags + tags = set() + for vision_tag in self._vision_vocabulary: + for candidate in (vision_tag, vision_tag.split("-")[0]): + language = OcrLanguageResolver.canonicalize_ocr_language( + candidate, raise_exception=False + ) + if language is not None: + tags.add(language.tag) + break + return sorted(tags) + + def map_ocr_language(self, language: OcrLanguage) -> str | list[str]: + if language.is_passthrough: + # One of Vision's own recognition languages (`en-US`), handed over as + # written rather than matched. + if language.native in self._vision_vocabulary: + assert language.native is not None + return language.native + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + ) + if language.is_multilingual: + # Vision has no multilingual model; an empty `lang` list is how its + # own automatic behaviour is selected. + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + detail="Apple Vision needs explicit languages.", + ) + # Vision's own vocabulary is BCP-47 with regions, so match rather than map. + code = OcrLanguageResolver.match_ocr_language(language, self._vision_vocabulary) + if code is None: + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + ) + return code + def __call__( self, conv_res: ConversionResult, page_batch: Iterable[Page] ) -> Iterable[Page]: @@ -94,7 +185,7 @@ def __call__( fname, recognition_level=self.options.recognition, framework=self.options.framework, - language_preference=self.options.lang, + language_preference=self._native_codes or None, ).recognize() im_width, im_height = high_res_image.size diff --git a/docling/models/stages/ocr/ppocr_languages.py b/docling/models/stages/ocr/ppocr_languages.py new file mode 100644 index 0000000000..ee120f06f3 --- /dev/null +++ b/docling/models/stages/ocr/ppocr_languages.py @@ -0,0 +1,151 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Canonical BCP-47 to PP-OCR recognizer codes. + +RapidOCR and the KServe v2 OCR client both address PP-OCR recognizers by the +same codes, so the mapping lives here rather than in either engine. RapidOCR +consults the installed `rapidocr` package for the authoritative PP-OCRv6 set and +falls back to the static copy below; the KServe client uses the static copy only, +so it never has to import `rapidocr`. + +The static code sets mirror the PP-OCR release notes summarised in +`docs/concepts/OCR.md`. They can drift from a newer `rapidocr`; this module is +their single owner. +""" + +from docling.utils.ocr_language import ( + OcrLanguage, + OcrLanguageResolver, +) + +# Recognition languages served by the PP-OCRv4 backbone (the torch fallback). +PPOCRV4_CODES = frozenset( + {"arabic", "cyrillic", "devanagari", "ka", "korean", "latin", "ta", "te"} +) + +# Recognition languages served by the PP-OCRv5 backbone. +PPOCRV5_CODES = frozenset( + { + "arabic", + "ch", + "cyrillic", + "devanagari", + "el", + "en", + "eslav", + "korean", + "latin", + "ta", + "te", + "th", + } +) + +# Static copy of the PP-OCRv6 recognition languages. RapidOCR prefers the set +# exported by the installed package; this is the offline/KServe fallback. +PPOCRV6_CODES = frozenset( + { + "af", "az", "bs", "ca", "ch", "chinese_cht", "cs", "cy", "da", "de", + "en", "es", "et", "eu", "fi", "fr", "french", "ga", "german", "gl", + "hr", "hu", "id", "is", "it", "japan", "ku", "la", "lb", "lt", "lv", + "mi", "ms", "mt", "nl", "no", "oc", "pl", "pt", "qu", "rm", "ro", + "rs_latin", "sk", "sl", "sq", "sv", "sw", "tl", "tr", "uz", "vi", + } +) # fmt: skip + +# PP-OCR code used when `lang` is left empty: the engine's own default +# recognizer, which is Simplified Chinese. +PPOCR_DEFAULT_CODE = "ch" + +# Canonical tag -> PP-OCR code, for the languages whose code is not simply the +# primary subtag. `None` marks a tag that must *not* fall through to the generic +# rules below, because the code that looks right means something else. +_CANONICAL_TO_CODE: dict[str, str | None] = { + "zh-Hans": "ch", + "zh-Hant": "chinese_cht", + "ja-Jpan": "japan", + "ko-Kore": "korean", + "sr-Latn": "rs_latin", + # `tl` is PP-OCR's code; BCP-47 canonicalizes Tagalog to `fil`. + "fil-Latn": "tl", + # PP-OCR serves East Slavic with a narrower recognizer than `cyrillic`. + "ru-Cyrl": "eslav", + "uk-Cyrl": "eslav", + "be-Cyrl": "eslav", + # PP-OCR's `ka` is Kannada; BCP-47 `ka` is Georgian. + "kn-Knda": "ka", + "ka-Geor": None, +} + +# ISO 15924 script -> PP-OCR script-family code. Internal routing only: users +# name a language and this finds the script-wide recognizer that covers it, for +# the many languages PP-OCR serves no other way. +_SCRIPT_TO_CODE: dict[str, str] = { + "Latn": "latin", + "Cyrl": "cyrillic", + "Arab": "arabic", + "Deva": "devanagari", +} + +# Reverse of the language table, for rendering a vocabulary back as tags. The +# script recognizers are not reversed: they are named back as themselves, which +# is what the user types to select one. +_CODE_TO_CANONICAL: dict[str, list[str]] = {} +for _tag, _token in _CANONICAL_TO_CODE.items(): + if _token is not None: + _CODE_TO_CANONICAL.setdefault(_token, []).append(_tag) + +# PP-OCRv6 codes that duplicate a language already reachable by its subtag. +_REDUNDANT_CODES = frozenset({"french", "german"}) + + +def ppocr_code(language: OcrLanguage, vocabulary: frozenset[str]) -> str | None: + """Map a canonical tag onto a PP-OCR code, or `None` if there is no model. + + `vocabulary` is the union of code sets the caller can actually reach, so + the resolution never returns a code the backend cannot serve. + """ + if language.is_passthrough: + # `arabic`, `cyrillic`: a recognizer named after a script, handed over as + # the user wrote it. + return language.native if language.native in vocabulary else None + if language.is_multilingual: + return None + + if language.bcp47 in _CANONICAL_TO_CODE: + code = _CANONICAL_TO_CODE[language.bcp47] + return code if code is not None and code in vocabulary else None + + # The primary subtag identifies the recognizer only when the language is + # written in its usual script: PP-OCR's `az` and `uz` are the Latin ones. + if language.has_default_script and language.bcp47_language in vocabulary: + return language.bcp47_language + + # PP-OCR serves many languages only through a script-wide recognizer: there + # is no `ar` or `hi` model, and on the PP-OCRv4 backbone most of the + # vocabulary is script models. This routing is internal -- users name a + # language and docling finds the recognizer that covers it. + family = _SCRIPT_TO_CODE.get(language.bcp47_script or "") + if family is not None and family in vocabulary: + return family + return None + + +def ppocr_supported_tags(vocabulary: frozenset[str]) -> list[str]: + """Render a PP-OCR code vocabulary back as the canonical tags it serves.""" + tags: set[str] = set() + for code in vocabulary: + if code in _REDUNDANT_CODES: + continue + if code in _CODE_TO_CANONICAL: + tags.update(_CODE_TO_CANONICAL[code]) + continue + language = OcrLanguageResolver.canonicalize_ocr_language( + code, raise_exception=False + ) + # `None` is a code that is not a language code and has no reverse + # entry; it is unreachable from a canonical tag anyway. + if language is not None: + tags.add(language.tag) + return sorted(tags) diff --git a/docling/models/stages/ocr/rapid_ocr_model.py b/docling/models/stages/ocr/rapid_ocr_model.py index d908105c2c..2abfb3c5e2 100644 --- a/docling/models/stages/ocr/rapid_ocr_model.py +++ b/docling/models/stages/ocr/rapid_ocr_model.py @@ -4,6 +4,7 @@ import logging from collections.abc import Iterable from dataclasses import dataclass +from functools import lru_cache from pathlib import Path from typing import TYPE_CHECKING, Type @@ -19,8 +20,22 @@ RapidOcrOptions, ) from docling.datamodel.settings import settings +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_ocr_model import BaseOcrModel +from docling.models.stages.ocr.ppocr_languages import ( + PPOCR_DEFAULT_CODE, + PPOCRV4_CODES, + PPOCRV5_CODES, + PPOCRV6_CODES, + ppocr_code, + ppocr_supported_tags, +) from docling.utils.accelerator_utils import decide_device +from docling.utils.ocr_language import ( + OcrLanguage, + OcrLanguageResolver, + OcrLanguageSupport, +) from docling.utils.profiling import TimeRecorder from docling.utils.utils import download_url_with_progress @@ -29,7 +44,8 @@ _log = logging.getLogger(__name__) -_RAPIDOCR_DEFAULT_LANGUAGE = "ch" +# Default OCR language as a canonical tag, for the prefetch entry points. +_RAPIDOCR_DEFAULT_LANGUAGE = "zh-Hans" # Recognition/detection model size for the PP-OCRv6 path; v4/v5 use "mobile". _RAPIDOCR_DET_MODEL_LANG = "ch" @@ -37,36 +53,11 @@ _RAPIDOCR_MODEL_TYPE = "small" _RAPIDOCR_V4V5_MODEL_TYPE = "mobile" -# Docling's default language names -> rapidocr language codes. -_DOCLING_LANG_NORMALIZE: dict[str, str] = {"chinese": "ch", "english": "en"} - # Inference backends docling supports. Must stay in sync with the `backend` Literal of # RapidOcrOptions in docling/datamodel/pipeline_options.py and with the mapping built in # _backend_to_engine_type(). _RAPIDOCR_BACKENDS: tuple[str, ...] = ("onnxruntime", "openvino", "paddle", "torch") -# Recognition languages served by the PP-OCRv4 backbone -_PPOCRV4_LANGS = frozenset( - {"arabic", "cyrillic", "devanagari", "ka", "korean", "latin", "ta", "te"} -) -# Recognition languages served by the PP-OCRv5 backbone -_PPOCRV5_LANGS = frozenset( - { - "arabic", - "ch", - "cyrillic", - "devanagari", - "el", - "en", - "eslav", - "korean", - "latin", - "ta", - "te", - "th", - } -) - @dataclass(frozen=True) class _RapidOcrArtifact: @@ -92,8 +83,8 @@ class _RapidOcrModelSpec: # Language exactly as the user wrote it user_lang: str | None = None - # Language code the rapidocr registry expects, after normalization and aliasing. - rapidocr_lang_token: str | None = None + # PP-OCR code the rapidocr registry expects, after normalization and aliasing. + rapidocr_code: str | None = None # PP-OCR backbone that the (backend, language) pair resolves to. ppocr_version: "OCRVersion | None" = None @@ -109,7 +100,7 @@ def _parse_rapidocr_model_spec(value: str) -> _RapidOcrModelSpec: if not separator or not backend or not lang or ":" in lang: raise ValueError( f"Invalid RapidOCR model spec {value!r}. " - "Expected ':', e.g. 'onnxruntime:th'." + "Expected ':', e.g. 'onnxruntime:th-Thai'." ) if backend not in _RAPIDOCR_BACKENDS: raise ValueError( @@ -118,7 +109,7 @@ def _parse_rapidocr_model_spec(value: str) -> _RapidOcrModelSpec: ) try: _resolve_rapidocr(lang, backend) - except ValueError as err: + except (ValueError, OcrLanguageNotSupportedError) as err: raise ValueError(f"Invalid RapidOCR model spec {value!r}: {err}") from err return _RapidOcrModelSpec(backend=backend, user_lang=lang) @@ -140,50 +131,76 @@ def _backend_to_engine_type(backend: str) -> "EngineType": return engine_types[backend] -def _resolve_rapidocr(lang: str, backend: str) -> _RapidOcrModelSpec: - """Map one requested language + backend onto a fully populated _RapidOcrModelSpec. +@lru_cache(maxsize=1) +def _installed_ppocrv6_codes() -> frozenset[str]: + """The PP-OCRv6 recognition languages, from the installed rapidocr if present. - - Prefer PP-OCRv6 (whose recognizer is multilingual and covers ~52 codes) - - Otherwise fall back to PP-OCRv4 for the torch backend or PP-OCRv5 for the others. - - Raises when the language cannot be served by the resolved backbone. + Falls back to docling's static copy so the mapping stays usable for the + KServe client, which does not require rapidocr. + """ + try: + from rapidocr.utils.model_resolver import PP_OCRV6_LANGS + except ImportError: + return PPOCRV6_CODES + return frozenset(PP_OCRV6_LANGS) - Callers pass a single language; reducing a multi-language request is up to them. + +@lru_cache(maxsize=len(_RAPIDOCR_BACKENDS)) +def _rapidocr_vocabulary(backend: str) -> frozenset[str]: + """PP-OCR codes a backend can serve: v6 plus its own v4/v5 fallback sets.""" + fallback = PPOCRV4_CODES if backend == "torch" else PPOCRV5_CODES | PPOCRV4_CODES + return _installed_ppocrv6_codes() | fallback + + +def _ppocr_version_for_code(code: str, backend: str) -> "OCRVersion": + """Which PP-OCR backbone serves a code on this backend. + + Prefers PP-OCRv6 (whose recognizer covers ~52 codes). Torch then falls back + to PP-OCRv4; the other backends try PP-OCRv5 first and PP-OCRv4 for the + codes v5 lacks -- `ka`, PP-OCR's Kannada, is the only one. """ - from rapidocr.utils.model_resolver import COMMON_LANG_ALIASES, PP_OCRV6_LANGS from rapidocr.utils.typings import OCRVersion - code = lang.strip().lower() - code = _DOCLING_LANG_NORMALIZE.get(code, code) - aliased = COMMON_LANG_ALIASES.get(code, code) - - if aliased in PP_OCRV6_LANGS: - version = OCRVersion.PPOCRV6 - elif backend == "torch": - if aliased not in _PPOCRV4_LANGS: - raise ValueError( - f"RapidOCR torch backend does not support language {lang!r}. " - f"Supported: {sorted(PP_OCRV6_LANGS | _PPOCRV4_LANGS)}." - ) - version = OCRVersion.PPOCRV4 - elif aliased in _PPOCRV5_LANGS: - version = OCRVersion.PPOCRV5 - else: - raise ValueError( - f"RapidOCR {backend} backend does not support language {lang!r}. " - f"Supported: {sorted(PP_OCRV6_LANGS | _PPOCRV5_LANGS)}." + if code in _installed_ppocrv6_codes(): + return OCRVersion.PPOCRV6 + if backend == "torch": + return OCRVersion.PPOCRV4 + return OCRVersion.PPOCRV5 if code in PPOCRV5_CODES else OCRVersion.PPOCRV4 + + +def _resolve_rapidocr(lang: str, backend: str) -> _RapidOcrModelSpec: + """Map one language + backend onto a fully populated _RapidOcrModelSpec. + + `lang` may be a BCP-47 tag or one of PP-OCR's own codes, matching what + `RapidOcrOptions.lang` accepts. + + Raises: + ValueError: `lang` is neither a PP-OCR code nor a valid BCP-47 tag. + OcrLanguageNotSupportedError: No PP-OCR recognizer serves it on `backend`. + + Callers pass a single language; reducing a multi-language request is up to them. + """ + language = OcrLanguageResolver.canonicalize_ocr_language(lang) + code = ppocr_code(language, _rapidocr_vocabulary(backend)) + if code is None: + raise OcrLanguageNotSupportedError( + f"RapidOCR (backend={backend})", + language.tag, + supported=ppocr_supported_tags(_rapidocr_vocabulary(backend)), ) + version = _ppocr_version_for_code(code, backend) _log.debug( - "RapidOCR resolved lang=%r backend=%r -> version=%s rec_lang=%r", + "RapidOCR resolved lang=%r backend=%r -> version=%s rec_code=%r", lang, backend, version.value, - aliased, + code, ) return _RapidOcrModelSpec( backend=backend, user_lang=lang, - rapidocr_lang_token=aliased, + rapidocr_code=code, ppocr_version=version, ) @@ -192,7 +209,7 @@ def _rapidocr_artifacts( target_dir: Path, engine: "EngineType", version: "OCRVersion", - rec_lang: str, + rec_code: str, *, need_det: bool = True, need_cls: bool = True, @@ -222,7 +239,7 @@ def _rapidocr_artifacts( engine, OCRVersion.PPOCRV4, TaskType.CLS, _RAPIDOCR_CLS_MODEL_LANG, cls_size ) if need_rec: - file_infos["rec"] = FileInfo(engine, version, TaskType.REC, rec_lang, size) + file_infos["rec"] = FileInfo(engine, version, TaskType.REC, rec_code, size) artifacts: dict[str, _RapidOcrArtifact] = {} for task, file_info in file_infos.items(): @@ -259,6 +276,8 @@ def _rapidocr_artifacts( class RapidOcrModel(BaseOcrModel): _model_repo_folder = "RapidOcr" + language_support = OcrLanguageSupport(multiple_languages=False) + def __init__( self, enabled: bool, @@ -276,6 +295,7 @@ def __init__( # multiplier for 72 dpi; the default 3.0 == 216 dpi. self.scale = self.options.scale + self._native_codes: list[str] = [] if self.enabled: try: @@ -296,23 +316,13 @@ def __init__( gpu_id = int(device.split(":")[1]) backend_enum = _backend_to_engine_type(self.options.backend) - # Reduce the user provided language list to one language + # One language, warn-and-truncate and coverage checks all happen here. + self._native_codes = self.resolve_ocr_languages() + rec_code = self._native_codes[0] lang = ( - self.options.lang[0] - if self.options.lang - else _RAPIDOCR_DEFAULT_LANGUAGE + self.languages[0].tag if self.languages else _RAPIDOCR_DEFAULT_LANGUAGE ) - if len(self.options.lang) > 1: - _log.warning( - "RapidOCR uses a single language; using %r and ignoring %r.", - lang, - self.options.lang[1:], - ) - resolved: _RapidOcrModelSpec = _resolve_rapidocr(lang, self.options.backend) - assert resolved.ppocr_version is not None - assert resolved.rapidocr_lang_token is not None - ppocr_version = resolved.ppocr_version - rec_lang = resolved.rapidocr_lang_token + ppocr_version = _ppocr_version_for_code(rec_code, self.options.backend) det_model_path = self.options.det_model_path cls_model_path = self.options.cls_model_path @@ -348,7 +358,7 @@ def __init__( target_dir, backend_enum, ppocr_version, - rec_lang, + rec_code, need_det=det_model_path is None, need_cls=cls_model_path is None, need_rec=rec_model_path is None, @@ -361,12 +371,12 @@ def __init__( ] if missing: listed = "\n".join(f" - {path}" for path in missing) - # `lang` is the user's own token, so the hint mirrors their config. + # `lang` is the canonical tag, which is what the prefetcher takes. raise FileNotFoundError( "RapidOCR artifacts not found or incomplete in artifacts_path.\n" f"Expected under: {target_dir}\n" f"Resolved: backend={self.options.backend} " - f"ppocr_version={ppocr_version.value} rec_lang={rec_lang}\n" + f"ppocr_version={ppocr_version.value} rec_code={rec_code}\n" f"Missing files:\n{listed}\n" "Prefetch them with:\n" f" docling-tools models download rapidocr " @@ -401,7 +411,7 @@ def __init__( lang_params["Cls.model_type"] = ModelType(_RAPIDOCR_V4V5_MODEL_TYPE) if rec_model_path is None: lang_params["Rec.ocr_version"] = ppocr_version - lang_params["Rec.lang_type"] = rec_lang + lang_params["Rec.lang_type"] = rec_code lang_params["Rec.model_type"] = size params = { @@ -454,6 +464,32 @@ def __init__( params=params, ) + def supported_ocr_languages(self) -> list[str]: + return ppocr_supported_tags(_rapidocr_vocabulary(self.options.backend)) + + def resolve_ocr_languages(self) -> list[str]: + # An empty `lang` list means "the engine's own default", which for PP-OCR + # is the Simplified Chinese recognizer. + if not self.languages: + return [PPOCR_DEFAULT_CODE] + return super().resolve_ocr_languages() + + def map_ocr_language(self, language: OcrLanguage) -> str: + code = ppocr_code(language, _rapidocr_vocabulary(self.options.backend)) + if code is None: + raise OcrLanguageNotSupportedError( + f"RapidOCR (backend={self.options.backend})", + language.tag, + supported=self.supported_ocr_languages(), + detail=( + "RapidOCR has no multilingual recognizer; name the languages " + "explicitly." + ) + if language.is_multilingual + else None, + ) + return code + @classmethod def download_models( cls, @@ -470,14 +506,14 @@ def download_models( resolved = _resolve_rapidocr(lang, backend) assert resolved.ppocr_version is not None - assert resolved.rapidocr_lang_token is not None + assert resolved.rapidocr_code is not None engine = _backend_to_engine_type(backend) for artifact in _rapidocr_artifacts( local_dir, engine, resolved.ppocr_version, - resolved.rapidocr_lang_token, + resolved.rapidocr_code, ).values(): for dest, url in artifact.files.items(): if dest.exists() and not force: diff --git a/docling/models/stages/ocr/tesseract_ocr_cli_model.py b/docling/models/stages/ocr/tesseract_ocr_cli_model.py index 74aad6c6e1..15c9d29123 100644 --- a/docling/models/stages/ocr/tesseract_ocr_cli_model.py +++ b/docling/models/stages/ocr/tesseract_ocr_cli_model.py @@ -25,12 +25,17 @@ TesseractCliOcrOptions, ) from docling.datamodel.settings import settings +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_ocr_model import BaseOcrModel -from docling.utils.ocr_utils import ( - map_tesseract_script, +from docling.models.stages.ocr.tesseract_utils import ( + installed_tesseract_tags, + language_to_tesseract_code, + osd_script_to_tesseract_code, parse_tesseract_orientation, tesseract_box_to_bounding_rectangle, + tesseract_vocabulary, ) +from docling.utils.ocr_language import OcrLanguage, OcrLanguageSupport from docling.utils.profiling import TimeRecorder _log = logging.getLogger(__name__) @@ -40,6 +45,8 @@ class TesseractOcrCliModel(BaseOcrModel): + language_support = OcrLanguageSupport(multiple_languages=True) + def __init__( self, enabled: bool, @@ -60,9 +67,11 @@ def __init__( self._name: Optional[str] = None self._version: Optional[str] = None - self._tesseract_languages: Optional[List[str]] = None - self._script_prefix: Optional[str] = None - self._is_auto: bool = "auto" in self.options.lang + self._tesseract_vocabulary: Optional[List[str]] = None + # No languages requested: Tesseract runs orientation and script + # detection per page and picks a `script/` reader from the result. + self._auto_script: bool = not self.languages + self._native_codes: List[str] = [] # Pre-validate and store sanitized subprocess arguments at construction time # so that all subsequent subprocess calls use only these already-validated values. @@ -72,15 +81,11 @@ def __init__( if self.options.path is not None else None ) - if self.options.lang: - for _lang_token in self.options.lang: - if _lang_token != "auto": - self._sanitize_lang(_lang_token) if self.enabled: try: self._get_name_and_version() - self._set_languages_and_prefix() + self._set_languages() except Exception as exc: raise RuntimeError( @@ -90,6 +95,38 @@ def __init__( "Alternatively, Docling has support for other OCR engines. See the documentation." ) + if self._auto_script and "osd" not in (self._tesseract_vocabulary or []): + raise ImportError( + "An empty OCR language list runs Tesseract's orientation and " + "script detection, which needs the 'osd' traineddata. Install " + "it (e.g. the tesseract-ocr-osd package) or name a language " + "explicitly in `ocr_options.lang`." + ) + + # Needs the installed language list, so it runs after the probe above. + self._native_codes = [ + self._sanitize_lang(lang) for lang in self.resolve_ocr_languages() + ] + + def supported_ocr_languages(self) -> List[str]: + return installed_tesseract_tags(self._tesseract_vocabulary or []) + + def map_ocr_language(self, language: OcrLanguage) -> str | List[str]: + assert self._tesseract_vocabulary is not None + name = language_to_tesseract_code(language) + if name is None or name not in self._tesseract_vocabulary: + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + detail=( + f"No traineddata file {name!r} is installed." + if name is not None + else "Tesseract has no traineddata for it." + ), + ) + return name + @staticmethod def _sanitize_lang(lang: str) -> str: """Validate and sanitize a Tesseract language identifier to prevent argument injection. @@ -170,16 +207,14 @@ def _run_tesseract(self, ifilename: str, osd: Optional[pd.DataFrame]): Run tesseract CLI """ cmd = [self._safe_tesseract_cmd] - if self._is_auto and osd is not None: + if self._auto_script and osd is not None: lang = self._parse_language(osd) if lang is not None: cmd.append("-l") cmd.append(self._sanitize_lang(lang)) - elif self.options.lang is not None and len(self.options.lang) > 0: + elif self._native_codes: cmd.append("-l") - cmd.append( - "+".join(self._sanitize_lang(lang) for lang in self.options.lang) - ) + cmd.append("+".join(self._native_codes)) if self._safe_tessdata_path is not None: cmd.append("--tessdata-dir") @@ -244,17 +279,17 @@ def _perform_osd(self, ifilename: str) -> pd.DataFrame: return df_detected def _parse_language(self, df_osd: pd.DataFrame) -> Optional[str]: - assert self._tesseract_languages is not None + assert self._tesseract_vocabulary is not None scripts = df_osd.loc[df_osd["key"] == "Script"].value.tolist() if len(scripts) == 0: _log.warning("Tesseract cannot detect the script of the page") return None - script = map_tesseract_script(scripts[0].strip()) - lang = f"{self._script_prefix}{script}" + script = scripts[0].strip() + lang = osd_script_to_tesseract_code(script) # Check if the detected language has been installed - if lang not in self._tesseract_languages: + if lang not in self._tesseract_vocabulary: msg = f"Tesseract detected the script '{script}' and language '{lang}'." msg += " However this language is not installed in your system and will be ignored." _log.warning(msg) @@ -265,9 +300,9 @@ def _parse_language(self, df_osd: pd.DataFrame) -> Optional[str]: ) return lang - def _set_languages_and_prefix(self): + def _set_languages(self): r""" - Read and set the languages installed in tesseract and decide the script prefix + Read and set the languages installed in tesseract """ # Get all languages cmd = [self._safe_tesseract_cmd, "--list-langs"] @@ -277,21 +312,7 @@ def _set_languages_and_prefix(self): ) decoded_data = output.stdout.decode("utf-8") df_list = pd.read_csv(io.StringIO(decoded_data), header=None) - # Tesseract prints script packs with the OS path separator, so on Windows - # `--list-langs` reports `script\Latin` rather than `script/Latin`. The - # forward-slash form is the one `_sanitize_lang` accepts and the one - # `tesseract -l` expects on every platform, so normalize on the way in. - self._tesseract_languages = [ - str(lang).replace("\\", "/") for lang in df_list[0].tolist()[1:] - ] - - # Decide the script prefix - if any(lang.startswith("script/") for lang in self._tesseract_languages): - script_prefix = "script/" - else: - script_prefix = "" - - self._script_prefix = script_prefix + self._tesseract_vocabulary = tesseract_vocabulary(df_list[0].tolist()[1:]) def __call__( self, conv_res: ConversionResult, page_batch: Iterable[Page] @@ -339,7 +360,7 @@ def __call__( ) # Skipping if OSD fail when in auto mode, otherwise proceed # to OCR in the hope OCR will succeed while OSD failed - if self._is_auto: + if self._auto_script: continue if doc_orientation != 0: high_res_image = high_res_image.rotate( diff --git a/docling/models/stages/ocr/tesseract_ocr_model.py b/docling/models/stages/ocr/tesseract_ocr_model.py index abc20e9499..4847bbed3c 100644 --- a/docling/models/stages/ocr/tesseract_ocr_model.py +++ b/docling/models/stages/ocr/tesseract_ocr_model.py @@ -18,18 +18,25 @@ TesseractOcrOptions, ) from docling.datamodel.settings import settings +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_ocr_model import BaseOcrModel -from docling.utils.ocr_utils import ( - map_tesseract_script, +from docling.models.stages.ocr.tesseract_utils import ( + installed_tesseract_tags, + language_to_tesseract_code, + osd_script_to_tesseract_code, parse_tesseract_orientation, tesseract_box_to_bounding_rectangle, + tesseract_vocabulary, ) +from docling.utils.ocr_language import OcrLanguage, OcrLanguageSupport from docling.utils.profiling import TimeRecorder _log = logging.getLogger(__name__) class TesseractOcrModel(BaseOcrModel): + language_support = OcrLanguageSupport(multiple_languages=True) + def __init__( self, enabled: bool, @@ -44,11 +51,15 @@ def __init__( accelerator_options=accelerator_options, ) self.options: TesseractOcrOptions - self._is_auto: bool = "auto" in self.options.lang + # No languages requested: Tesseract runs orientation and script + # detection per page and picks a `script/` reader from the result. + self._auto_script: bool = not self.languages # multiplier for 72 dpi; the default 3.0 == 216 dpi. self.scale = self.options.scale self.reader = None self.script_readers: dict[str, tesserocr.PyTessBaseAPI] = {} + self._tesseract_vocabulary: list[str] = [] + self._native_codes: list[str] = [] if self.enabled: install_errmsg = ( @@ -76,18 +87,24 @@ def __init__( except Exception: raise ImportError(install_errmsg) - _, self._tesserocr_languages = tesserocr.get_languages() - if not self._tesserocr_languages: + _, codes = tesserocr.get_languages() + self._tesseract_vocabulary = tesseract_vocabulary(codes) + if not self._tesseract_vocabulary: raise ImportError(missing_langs_errmsg) # Initialize the tesseractAPI _log.debug("Initializing TesserOCR: %s", tesseract_version) - lang = "+".join(self.options.lang) - if any(lang.startswith("script/") for lang in self._tesserocr_languages): - self.script_prefix = "script/" - else: - self.script_prefix = "" + if self._auto_script and "osd" not in self._tesseract_vocabulary: + raise ImportError( + "An empty OCR language list runs Tesseract's orientation and " + "script detection, which needs the 'osd' traineddata. Install " + "it (e.g. the tesseract-ocr-osd package) or name a language " + "explicitly in `ocr_options.lang`." + ) + + # Needs the installed language list and the prefix, so it runs here. + self._native_codes = self.resolve_ocr_languages() tesserocr_kwargs = { "init": True, @@ -103,11 +120,12 @@ def __init__( main_psm = ( self.options.psm if self.options.psm is not None else tesserocr.PSM.AUTO ) - if lang == "auto": + if self._auto_script: + # No `lang`: the per-page OSD pass picks a script reader instead. self.reader = tesserocr.PyTessBaseAPI(psm=main_psm, **tesserocr_kwargs) else: self.reader = tesserocr.PyTessBaseAPI( - lang=lang, + lang="+".join(self._native_codes), psm=main_psm, **tesserocr_kwargs, ) @@ -117,12 +135,30 @@ def __init__( ) self.reader_RIL = tesserocr.RIL + def supported_ocr_languages(self) -> list[str]: + return installed_tesseract_tags(self._tesseract_vocabulary) + + def map_ocr_language(self, language: OcrLanguage) -> str | list[str]: + name = language_to_tesseract_code(language) + if name is None or name not in self._tesseract_vocabulary: + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + detail=( + f"No traineddata file {name!r} is installed." + if name is not None + else "Tesseract has no traineddata for it." + ), + ) + return name + def __del__(self): if self.reader is not None: # Finalize the tesseractAPI self.reader.End() - for script in self.script_readers: - self.script_readers[script].End() + for reader in self.script_readers.values(): + reader.End() def __call__( self, conv_res: ConversionResult, page_batch: Iterable[Page] @@ -139,7 +175,7 @@ def __call__( with TimeRecorder(conv_res, "ocr"): assert self.reader is not None assert self.osd_reader is not None - assert self._tesserocr_languages is not None + assert self._tesseract_vocabulary is not None ocr_rects = self.get_ocr_rects(page) @@ -169,7 +205,7 @@ def __call__( ) # Skipping if OSD fail when in auto mode, otherwise proceed # to OCR in the hope OCR will succeed while OSD failed - if self._is_auto: + if self._auto_script: continue else: doc_orientation = parse_tesseract_orientation( @@ -179,32 +215,29 @@ def __call__( high_res_image = high_res_image.rotate( -doc_orientation, expand=True ) - if self._is_auto: + if self._auto_script: script = osd["script_name"] - script = map_tesseract_script(script) - lang = f"{self.script_prefix}{script}" + lang = osd_script_to_tesseract_code(script) # Check if the detected language is present in the system - if lang not in self._tesserocr_languages: + if lang not in self._tesseract_vocabulary: msg = f"Tesseract detected the script '{script}' and language '{lang}'." msg += " However this language is not installed in your system and will be ignored." _log.warning(msg) else: - if script not in self.script_readers: + if lang not in self.script_readers: import tesserocr - self.script_readers[script] = ( - tesserocr.PyTessBaseAPI( - path=self.reader.GetDatapath(), - lang=lang, - psm=self.options.psm - if self.options.psm is not None - else tesserocr.PSM.AUTO, - init=True, - oem=tesserocr.OEM.DEFAULT, - ) + self.script_readers[lang] = tesserocr.PyTessBaseAPI( + path=self.reader.GetDatapath(), + lang=lang, + psm=self.options.psm + if self.options.psm is not None + else tesserocr.PSM.AUTO, + init=True, + oem=tesserocr.OEM.DEFAULT, ) - local_reader = self.script_readers[script] + local_reader = self.script_readers[lang] local_reader.SetImage(high_res_image) boxes = local_reader.GetComponentImages( diff --git a/docling/models/stages/ocr/tesseract_utils.py b/docling/models/stages/ocr/tesseract_utils.py new file mode 100644 index 0000000000..73a402fb51 --- /dev/null +++ b/docling/models/stages/ocr/tesseract_utils.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Tessdata names, orientation and box geometry, shared by both Tesseract models. + +`tesseract_ocr_model.py` (the tesserocr bindings) and `tesseract_ocr_cli_model.py` +drive the same engine through different front-ends, so everything that is about +Tesseract itself rather than about either front-end lives here -- the same reason +`ppocr_languages.py` sits beside the two engines that speak PP-OCR. +""" + +from collections.abc import Sequence +from typing import Optional, Tuple + +import langcodes +from docling_core.types.doc import BoundingBox, CoordOrigin +from docling_core.types.doc.page import BoundingRectangle + +from docling.utils.ocr_language import ( + OcrLanguage, + OcrLanguageResolver, +) +from docling.utils.orientation import CLIPPED_ORIENTATIONS, rotate_bounding_box + +# Tessdata files that are not recognizers: `osd` is the orientation-and-script +# detector, `equ` the equation model. Neither is a language anyone can ask for. +_NON_LANGUAGE_TRAINEDDATA = frozenset({"osd", "equ"}) + + +# Canonical tag -> tessdata file, where Tesseract deviates from ISO 639-2/T +# Everything else is handled by `.to_alpha3(variant="T")`. +_TESSERACT_DEVIATIONAL_CODES: dict[str, str] = { + "zh-Hans": "chi_sim", + "zh-Hant": "chi_tra", + "sr-Cyrl": "srp", + "sr-Latn": "srp_latn", + "az-Cyrl": "aze_cyrl", + "az-Latn": "aze", + "uz-Cyrl": "uzb_cyrl", + "uz-Latn": "uzb", + "ku-Latn": "kmr", + "nb-Latn": "nor", + "nn-Latn": "nor", + "no-Latn": "nor", + # Fraktur has its own traineddata; `to_alpha3()` would flatten it to `deu`. + "de-Latf": "deu_latf", +} + +_DEVIATIONAL_CODE_TO_CANONICAL: dict[str, str] = { + name: tag for tag, name in _TESSERACT_DEVIATIONAL_CODES.items() +} + + +# Prefix of the tessdata script-family files, e.g. `script/Cyrillic`. The bare +# script name is deliberately *not* accepted as a language: `Lao` is a tessdata +# script file and also a valid BCP-47 primary subtag. +_TESSERACT_SCRIPT_FILE_PREFIX = "script/" + + +def tesseract_vocabulary(codes: Sequence[str]) -> list[str]: + r"""The traineddata names an install reports, normalized. + + Tesseract spells script packs with the OS path separator, so on Windows it + reports `script\Latin`. Both front-ends see it: `tesseract --list-langs` + prints `GetAvailableLanguagesAsVector()` and `tesserocr.get_languages()` + calls the same API. The forward-slash form is what `tesseract -l` expects on + every platform, and the only one `_sanitize_lang` accepts. + """ + return [str(code).replace("\\", "/") for code in codes] + + +def osd_script_to_tesseract_code(script: str) -> str: + """The tessdata file an OSD-detected script selects: `Katakana` -> `script/Japanese`. + + OSD reports a script, never a language, out of the fixed set its traineddata + was built on, and every one of those names has a `script/` file once the few + that are not spelled like their file are folded onto it. + """ + if script == "Katakana" or script == "Hiragana": + script = "Japanese" + elif script == "Han": + script = "HanS" + elif script == "Korean": + script = "Hangul" + return f"{_TESSERACT_SCRIPT_FILE_PREFIX}{script}" + + +def language_to_tesseract_code(language: OcrLanguage) -> str | None: + """Map an OcrLanguage object to a tesseract code""" + if language.native is not None: + return language.native + if language.is_multilingual: + return None + if language.bcp47 in _TESSERACT_DEVIATIONAL_CODES: + return _TESSERACT_DEVIATIONAL_CODES[language.bcp47] + # Tesseract's vocabulary *is* ISO 639-2/T: deu, fra, ell, ces, kat. + assert language.bcp47_language is not None + return langcodes.Language.get(language.bcp47_language).to_alpha3(variant="T") + + +def installed_tesseract_tags(codes: Sequence[str]) -> list[str]: + """ + The tags this install can serve. This can be either a canonical BCP47 tag or a native tesseract + """ + tags = set() + for code in codes: + if code in _NON_LANGUAGE_TRAINEDDATA: + continue + # First resolve against the deviational codes + tag = _DEVIATIONAL_CODE_TO_CANONICAL.get(code, code) + + # Try to canonicalize or receive a None language + language = OcrLanguageResolver.canonicalize_ocr_language( + tag, raise_exception=False + ) + # Check if it is a native code + if language is None or language_to_tesseract_code(language) != code: + language = OcrLanguage(native=code) + tags.add(language.tag) + return sorted(tags) + + +def parse_tesseract_orientation(orientation: str) -> int: + # Tesseract orientation is [0, 90, 180, 270] clockwise, bounding rectangle angles + # are [0, 360[ counterclockwise + parsed = int(orientation) + if parsed not in CLIPPED_ORIENTATIONS: + msg = ( + f"invalid tesseract document orientation {orientation}, " + f"expected orientation: {sorted(CLIPPED_ORIENTATIONS)}" + ) + raise ValueError(msg) + parsed = -parsed + parsed %= 360 + return parsed + + +def tesseract_box_to_bounding_rectangle( + bbox: BoundingBox, + *, + original_offset: Optional[BoundingBox] = None, + scale: float, + orientation: int, + im_size: Tuple[int, int], +) -> BoundingRectangle: + # box is in the top, left, height, width format, top left coordinates + rect = rotate_bounding_box(bbox, angle=orientation, im_size=im_size) + rect = BoundingRectangle( + r_x0=rect.r_x0 / scale, + r_y0=rect.r_y0 / scale, + r_x1=rect.r_x1 / scale, + r_y1=rect.r_y1 / scale, + r_x2=rect.r_x2 / scale, + r_y2=rect.r_y2 / scale, + r_x3=rect.r_x3 / scale, + r_y3=rect.r_y3 / scale, + coord_origin=CoordOrigin.TOPLEFT, + ) + if original_offset is not None: + if original_offset.coord_origin is not CoordOrigin.TOPLEFT: + msg = f"expected coordinate origin to be {CoordOrigin.TOPLEFT.value}" + raise ValueError(msg) + if original_offset is not None: + rect.r_x0 += original_offset.l + rect.r_x1 += original_offset.l + rect.r_x2 += original_offset.l + rect.r_x3 += original_offset.l + rect.r_y0 += original_offset.t + rect.r_y1 += original_offset.t + rect.r_y2 += original_offset.t + rect.r_y3 += original_offset.t + return rect diff --git a/docling/utils/model_downloader.py b/docling/utils/model_downloader.py index c9b673e0ad..2fe2e75e10 100644 --- a/docling/utils/model_downloader.py +++ b/docling/utils/model_downloader.py @@ -22,6 +22,7 @@ from docling.models.stages.ocr.easyocr_model import ( EasyOcrModel, _resolve_easyocr_recognition_models, + resolve_easyocr_codes, ) from docling.models.stages.ocr.nemotron_ocr_model import ( NemotronOcrModel, @@ -72,7 +73,7 @@ def download_models( with_rapidocr: bool = True, rapidocr_models: Optional[list[str]] = None, with_easyocr: bool = False, - easyocr_languages: Optional[list[str]] = None, + easyocr_languages: Optional[list[str]] = None, # BCP-47 tags with_nemotron_ocr: bool = False, ): if easyocr_languages is not None and not with_easyocr: @@ -83,7 +84,7 @@ def download_models( easyocr_recognition_models = ["english_g2", "latin_g2"] if easyocr_languages is not None: easyocr_recognition_models = _resolve_easyocr_recognition_models( - easyocr_languages + resolve_easyocr_codes(easyocr_languages) ) if output_dir is None: diff --git a/docling/utils/ocr_language.py b/docling/utils/ocr_language.py new file mode 100644 index 0000000000..117e055e53 --- /dev/null +++ b/docling/utils/ocr_language.py @@ -0,0 +1,308 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Canonicalization of OCR language requests to BCP-47 (RFC 5646). + +The user can provide as input language either a supported BCP-47 code or a language native to the +OCR engine. + +Docling reduces every request to a `(language, script)` pair. +Per-engine adapters translate that pair into the engine's own notation +(see `docling.models.base_ocr_model.BaseOcrModel.map_ocr_language`). + +Region is discarded once it has inferred the script: `zh-CN` and `zh-Hans` are +the same recognizer, and `de-DE` vs `de-AT` is a distinction no OCR engine +docling supports can act on. +""" + +import logging +from collections.abc import Sequence +from functools import lru_cache +from typing import Literal, overload + +import langcodes +from pydantic import BaseModel, ConfigDict + +_log = logging.getLogger(__name__) + + +class OcrLanguageSupport(BaseModel): + """Static, engine-declared language capabilities. + + Attributes: + multiple_languages: Whether the engine can run several languages at + once. `False` marks a single-language engine, whose extra tags are + dropped with a warning. + """ + + model_config = ConfigDict(frozen=True) + + multiple_languages: bool = False + + +class OcrLanguage(BaseModel): + """One canonicalized OCR language request: a BCP-47 (language, script) pair. + + Attributes: + bcp47_language: Primary subtag, lowercase. May be the reserved subtag + `mul`. + bcp47_script: ISO 15924 script code in title case. `None` only for the + bare reserved tags. + native: An engine's own token, stripped of the `native:` prefix, for + the models no `(bcp47_language, bcp47_script)` pair can name -- + PP-OCR's script recognizers (`arabic`, `cyrillic`) and Tesseract's + `script/` files. Set only for a passthrough, where it excludes + `bcp47_language` and `bcp47_script` and is what `tag` re-prefixes; + `None` for every ordinary BCP-47 request. + """ + + model_config = ConfigDict(frozen=True) + + bcp47_language: str | None = None + bcp47_script: str | None = None + native: str | None = None + + @property + def tag(self) -> str: + """How this request is written back into `OcrOptions.lang`. + + A canonical BCP-47 tag (`de-Latn`, `mul`), or, for a passthrough, the + engine's own token behind the `native:` prefix. Re-attaching the prefix + is what keeps `lang` idempotent: revalidating `["native:arabic"]` must + not move it. + """ + if self.native is not None: + return f"{OcrLanguageResolver._NATIVE_PREFIX}{self.native}" + return self.bcp47 + + @property + def bcp47(self) -> str: + """The `(language, script)` pair written as a BCP-47 tag. + + What an engine's own table is keyed on. Unlike `tag` it never carries + the `native:` prefix, so it is empty for a passthrough, which names no + language at all. + """ + return ( + f"{self.bcp47_language}-{self.bcp47_script}" + if self.bcp47_script + else self.bcp47_language or "" + ) + + @property + def is_passthrough(self) -> bool: + """An engine token that no `(language, script)` pair can express. + + PP-OCR's script recognizers (`arabic`, `cyrillic`) and Tesseract's + `script/` files: real models, named after a script rather than a + language, and handed to the engine untouched. + """ + return self.native is not None + + @property + def is_multilingual(self) -> bool: + return self.bcp47_language == OcrLanguageResolver.MULTIPLE + + @property + def has_default_script(self) -> bool: + """Whether `script` is the script CLDR considers likely for `language`. + + `de-Latn` and `en-Latn` do; `az-Cyrl` and `uz-Cyrl` do not. Engines use + this to decide whether the primary subtag alone still identifies the + right recognizer. + """ + if ( + self.bcp47_language is None + or self.bcp47_language == OcrLanguageResolver.MULTIPLE + ): + return False + return self.bcp47_script == OcrLanguageResolver._default_script_for_language( + self.bcp47_language + ) + + def __str__(self) -> str: + return self.tag + + +class OcrLanguageResolver: + """Canonicalizes user-supplied OCR language tokens into `OcrLanguage`. + + A namespace rather than an object: every entry point is a `@staticmethod`, + the vocabularies and legacy tables are class variables, and the expensive + steps memoize on their arguments alone. + """ + + # Multiple languages: the engine's broadest multilingual model. + MULTIPLE = "mul" + + _OCR_DOCS_URL = "https://docling-project.github.io/docling/concepts/OCR/" + + # When docling's input language has the following prefix, it passthrough the OCR engine + _NATIVE_PREFIX = "native:" + + # BCP-47's "undetermined". Docling does *not* accept it as an OCR language + _UNDETERMINED = "und" + + @staticmethod + def canonicalize_ocr_languages(values: Sequence[str]) -> list[OcrLanguage]: + """Canonicalize a list of language requests, enforcing the reserved-tag rule. + + Callers that store strings -- `OcrOptions.lang`, the remote CLI -- read + `.tag` off each result. + + An empty list is valid and means "the engine's own default"; every + engine decides what that is, and for Tesseract it is per-page script + detection. + + Duplicates are dropped, preserving the order the user wrote + + Raise a ValueError if the "multiple" language has been used together with other languages + """ + languages: list[OcrLanguage] = [] + for value in values: + language = OcrLanguageResolver.canonicalize_ocr_language(value) + if language not in languages: + languages.append(language) + + # Validate if the "multiple" language has been used together with other languages + reserved = [ + lang.tag + for lang in languages + if lang.bcp47_language == OcrLanguageResolver.MULTIPLE + and lang.bcp47_script is None + ] + if reserved and len(languages) > 1: + raise ValueError( + f"The reserved OCR language tag {reserved[0]!r} must be used on its " + f"own, but it was combined with " + f"{[lang.tag for lang in languages if lang.tag != reserved[0]]}." + ) + return languages + + @overload + @staticmethod + def canonicalize_ocr_language( + value: str, + *, + raise_exception: Literal[True] = True, + ) -> OcrLanguage: ... + + @overload + @staticmethod + def canonicalize_ocr_language( + value: str, + *, + raise_exception: Literal[False], + ) -> "OcrLanguage | None": ... + + @staticmethod + def canonicalize_ocr_language( + value: str, + *, + raise_exception: bool = True, + ) -> "OcrLanguage | None": + """Canonicalize one user-supplied OCR language. + + `raise_exception`: When `False`, a value that cannot be resolved + returns `None` instead of raising. + + Raises: + ValueError + """ + try: + token = value.strip() + if not token: + raise OcrLanguageResolver._invalid(value, "the value is empty.") + lowered = token.lower() + + # Passthrough the native tokens + if lowered.startswith(OcrLanguageResolver._NATIVE_PREFIX): + native = token[len(OcrLanguageResolver._NATIVE_PREFIX) :].strip() + return OcrLanguage(native=native) + + # Expect a BCP47 input + return OcrLanguageResolver._parse_bcp47(lowered) + except ValueError: + if raise_exception: + raise + return None + + @staticmethod + def match_ocr_language( + language: OcrLanguage, supported: Sequence[str], *, max_distance: int = 10 + ) -> str | None: + """Pick the closest entry of a BCP-47-ish engine vocabulary, or `None`. + + Only useful where the engine's own vocabulary is itself BCP-47 and + carries regions (Apple Vision). Everything else should use an explicit + table. + """ + if not supported: + return None + # The BCP-47 pair, never `tag`: a passthrough's engine code is not a tag + # and has nothing to match against. + match, distance = langcodes.closest_match( + language.bcp47, list(supported), max_distance=max_distance + ) + return None if match == OcrLanguageResolver._UNDETERMINED else match + + @staticmethod + @lru_cache(maxsize=256) + def _default_script_for_language(bcp47_language: str) -> str | None: + """The script CLDR likely-subtags associate with a primary subtag.""" + try: + return langcodes.Language.get(bcp47_language).maximize().script + except langcodes.LanguageTagError: + return None + + @staticmethod + def _invalid(value: str, reason: str) -> ValueError: + return ValueError( + f"Invalid OCR language {value!r}. Docling uses BCP-47 language tags; " + f"{reason} See {OcrLanguageResolver._OCR_DOCS_URL}" + ) + + @staticmethod + @lru_cache(maxsize=256) + def _parse_bcp47(bcp47_tobe: str) -> OcrLanguage: + """Canonicalize one value as a plain BCP-47 tag, with no engine vocabulary.""" + + if bcp47_tobe == OcrLanguageResolver.MULTIPLE: + return OcrLanguage(bcp47_language=bcp47_tobe) + + try: + parsed = langcodes.Language.get(bcp47_tobe, normalize=True) + except langcodes.LanguageTagError as err: + raise OcrLanguageResolver._invalid(bcp47_tobe, f"{err}.") from err + + if not parsed.is_valid(): + raise OcrLanguageResolver._invalid( + bcp47_tobe, "the tag is not registered with IANA." + ) + + if bcp47_tobe == "zxx": + # Docling has no recognizer for "no linguistic content" + raise OcrLanguageResolver._invalid( + bcp47_tobe, + "it names 'no linguistic content'. To skip OCR, turn it off " + "instead: `--no-ocr` on the CLI, or `do_ocr=False` in the " + "pipeline options.", + ) + + if ( + parsed.language is None + or parsed.language == OcrLanguageResolver._UNDETERMINED + ): + # Docling has no undetermined language and no script families + raise OcrLanguageResolver._invalid( + bcp47_tobe, + "docling has no 'undetermined' language and no script families: " + "leave the OCR language list empty to let the engine choose " + "(which is how Tesseract's per-page script detection is enabled), " + "or name a language written in the script you want.", + ) + + # An explicit script wins, so `de-Latf` keeps Fraktur; only a tag written + # without one falls back to CLDR's likely script, sending `de` to `de-Latn`. + script = parsed.script or parsed.maximize().script + return OcrLanguage(bcp47_language=parsed.language, bcp47_script=script) diff --git a/docling/utils/ocr_utils.py b/docling/utils/ocr_utils.py deleted file mode 100644 index a1a21cd7ac..0000000000 --- a/docling/utils/ocr_utils.py +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-FileCopyrightText: The Docling Contributors -# SPDX-License-Identifier: MIT - -from typing import Optional, Tuple - -from docling_core.types.doc import BoundingBox, CoordOrigin -from docling_core.types.doc.page import BoundingRectangle - -from docling.utils.orientation import CLIPPED_ORIENTATIONS, rotate_bounding_box - - -def map_tesseract_script(script: str) -> str: - r""" """ - if script == "Katakana" or script == "Hiragana": - script = "Japanese" - elif script == "Han": - script = "HanS" - elif script == "Korean": - script = "Hangul" - return script - - -def parse_tesseract_orientation(orientation: str) -> int: - # Tesseract orientation is [0, 90, 180, 270] clockwise, bounding rectangle angles - # are [0, 360[ counterclockwise - parsed = int(orientation) - if parsed not in CLIPPED_ORIENTATIONS: - msg = ( - f"invalid tesseract document orientation {orientation}, " - f"expected orientation: {sorted(CLIPPED_ORIENTATIONS)}" - ) - raise ValueError(msg) - parsed = -parsed - parsed %= 360 - return parsed - - -def tesseract_box_to_bounding_rectangle( - bbox: BoundingBox, - *, - original_offset: Optional[BoundingBox] = None, - scale: float, - orientation: int, - im_size: Tuple[int, int], -) -> BoundingRectangle: - # box is in the top, left, height, width format, top left coordinates - rect = rotate_bounding_box(bbox, angle=orientation, im_size=im_size) - rect = BoundingRectangle( - r_x0=rect.r_x0 / scale, - r_y0=rect.r_y0 / scale, - r_x1=rect.r_x1 / scale, - r_y1=rect.r_y1 / scale, - r_x2=rect.r_x2 / scale, - r_y2=rect.r_y2 / scale, - r_x3=rect.r_x3 / scale, - r_y3=rect.r_y3 / scale, - coord_origin=CoordOrigin.TOPLEFT, - ) - if original_offset is not None: - if original_offset.coord_origin is not CoordOrigin.TOPLEFT: - msg = f"expected coordinate origin to be {CoordOrigin.TOPLEFT.value}" - raise ValueError(msg) - if original_offset is not None: - rect.r_x0 += original_offset.l - rect.r_x1 += original_offset.l - rect.r_x2 += original_offset.l - rect.r_x3 += original_offset.l - rect.r_y0 += original_offset.t - rect.r_y1 += original_offset.t - rect.r_y2 += original_offset.t - rect.r_y3 += original_offset.t - return rect diff --git a/docs/concepts/OCR.md b/docs/concepts/OCR.md index 79f8fcb5d9..4b9dc60213 100644 --- a/docs/concepts/OCR.md +++ b/docs/concepts/OCR.md @@ -1,4 +1,4 @@ -# OCR engines in Docling +# OCR in Docling ## Overview @@ -11,11 +11,167 @@ Docling supports multiple OCR engines that can be installed as extra packages: - [tesseract-CLI](https://github.com/tesseract-ocr/tesseract) - [tesserocr](https://github.com/sirfz/tesserocr) +## Language selection + +Every OCR engine takes its languages through the same field, `OcrOptions.lang`, and every engine +takes them in the same vocabulary: **BCP-47 (RFC 5646) language tags**, the notation behind `en`, +`de-DE` and `zh-Hant`. Docling canonicalizes each tag to a `(language, script)` pair and each engine +translates that pair into its own notation. + +```python +from docling.datamodel.pipeline_options import TesseractCliOcrOptions + +TesseractCliOcrOptions(lang=["de", "en"]) # -> tesseract -l deu+eng +``` + +Canonicalization drops the region once it has told us the script, because no OCR engine +distinguishes `de-DE` from `de-AT`: + +| You write | Docling stores | Why | +| ----------------------------- | -------------- | ----------------------------------------- | +| `de`, `de-DE`, `deu`, `ger` | `de-Latn` | ISO 639-1/2/3 fold together; region drops | +| `en`, `en-US`, `eng` | `en-Latn` | | +| `zh`, `zh-CN`, `zho` | `zh-Hans` | Simplified is the likely script for `zh` | +| `zh-TW`, `zh-HK`, `zh-Hant` | `zh-Hant` | Traditional | +| `sr` | `sr-Cyrl` | Serbian defaults to Cyrillic | +| `sr-Latn` | `sr-Latn` | Same language, a different model | +| `pa`, `pa-IN` | `pa-Guru` | Gurmukhi | +| `pa-PK` | `pa-Arab` | Shahmukhi | + +The list order is preference order. Duplicates collapse, so `["de", "de-AT"]` is one language. + +### The reserved tag + +One tag carries engine-independent meaning, and it must be used **alone**. + +| Tag | Meaning | Behaviour | +| ----- | ------------------ | ---------------------------------------- | +| `mul` | multiple languages | The engine's broadest multilingual model | + +An **empty list** is how you say "let the engine decide", and a script is named by naming a +language written in it. To skip OCR altogether, turn the stage off — `--no-ocr` on the CLI, +`do_ocr=False` in the pipeline options. + +What each engine does with an empty `lang` and with `mul`: + +| Engine | `lang=[]` | `mul` | +| ----------------- | ---------------------------------- | --------------------------- | +| Tesseract (both) | Per-page orientation and script | Error | +| | detection; needs the `osd` file | | +| EasyOCR | English (`en`) | Error -- list the languages | +| RapidOCR | The Simplified Chinese default | Error -- list the languages | +| KServe | Sends `en` | Sent to the server verbatim | +| Nemotron-OCR | The English model | The multilingual model | +| ocrmac | Vision's own automatic behaviour | Error | + +On the CLI, omitting `--ocr-lang` applies the engine's default languages; an empty value, +`--ocr-lang ""`, is how you ask for the `lang=[]` column above. + +Engines that ship a recognizer named after a script rather than a language expose it under +that engine's own token: `latin`, `cyrillic`, `arabic` and `devanagari` for RapidOCR, and +`script/` files such as `script/Cyrillic` for Tesseract. They are engine +vocabulary, not portable tags, so they are only accepted by the engine that defines them. + +The KServe client is the exception to everything in this page: it canonicalizes nothing. Only the +deployed model knows which languages it serves, so `lang` is neither validated nor mapped -- the +first entry is sent to the server exactly as written, and the rest are dropped with a warning. Use +the codes your deployment expects (`english`, `chinese`, `ch`, ...); a BCP-47 tag is only right if +the server itself speaks BCP-47, and the `native:` prefix has no meaning there. + +### Engine-native language codes + +You can also write the language codes of the engine you selected, and they mean what that engine +means by them. Docling canonicalizes them on the way in, so `lang` always ends up holding BCP-47: + +```python +from docling.datamodel.pipeline_options import RapidOcrOptions + +RapidOcrOptions(lang=["ch"]).lang # -> ["zh-Hans"] +``` + +| Engine | Codes accepted in addition to BCP-47 | +| ----------------- | ------------------------------------------------------------------ | +| RapidOCR | PP-OCR tokens: `ch`, `chinese_cht`, `japan`, `korean`, `ka`, | +| | `eslav`, `latin`, `cyrillic`, `arabic`, `devanagari`, `rs_latin`, | +| | `french`, `german` -- the last two always canonicalize to `fr` | +| | and `de` | +| Tesseract (both) | tessdata names: `chi_sim`, `chi_tra`, `srp_latn`, `aze_cyrl`, | +| | `uzb_cyrl`, `deu_latf`, `frk`, and any `script/` file | +| EasyOCR | EasyOCR codes: `ch_sim`, `ch_tra`, `rs_latin`, `rs_cyrillic`, | +| | `tjk`, `ang`, `mah`, `tab` | +| Nemotron-OCR | `english`, `multilingual` | +| ocrmac | none needed -- Vision's own vocabulary already is BCP-47 | + +Only codes the engine spells differently from ISO 639 are listed. The rest of every engine's +vocabulary (`deu`, `fra`, `ru`, `ta`, `en-US`, ...) is valid BCP-47 already and has always worked. + +A code belongs to **one** engine. Asking RapidOCR for Tesseract's `chi_sim` is an error naming the +tag to write instead, because reading another engine's vocabulary would make the same string mean +different things depending on a setting elsewhere in your config. + +### Common surprises + +Six codes are a legitimate BCP-47 tag for one language and an engine's own name for a different one. +The engine that owns the code wins, which is what makes existing configurations keep working: + +| Code | Engine means | BCP-47 means | Owned by | +| ----- | -------------------- | ------------ | --------- | +| `ch` | Chinese Simplified | Chamorro | PP-OCR | +| `ka` | Kannada | Georgian | PP-OCR | +| `ang` | Angika | Old English | EasyOCR | +| `mah` | Magahi | Marshallese | EasyOCR | +| `tab` | Tabasaran (Cyrillic) | Tabasaran | EasyOCR | +| `frk` | German Fraktur | Frankish | Tesseract | + +This is safe rather than lucky: no engine ships a recognizer for any of the shadowed readings, so +nothing that was reachable before becomes unreachable. A test enforces that property, so an engine +that later gains one of those models turns the code into a reported ambiguity instead of a silent +wrong answer. + +To ask for the BCP-47 meaning anyway, write the script out. The tables above are keyed on the bare +code, so a qualified tag bypasses them: + +```python +RapidOcrOptions(lang=["ch-Latn"]) # Chamorro -- and so, an honest "no model" error +RapidOcrOptions(lang=["ka-Geor"]) # Georgian, not Kannada +EasyOcrOptions(lang=["ang-Latn"]) # Old English, not Angika +``` + +With no engine selected -- `docling convert-remote`, or `--ocr-engine auto` before it has picked one +-- there is nothing to prefer an engine's reading over the standard one, so only the codes every +engine agrees on carry their engine meaning. `ch` is refused there and names `zh-Hans` in the +message; the other five parse as ordinary BCP-47 and mean the *BCP-47* column above, so `ka` is +Georgian and `ang` is Old English. Name the engine, or write the qualified tag, to be explicit. + +### Codes with no tag + +A few native codes name a model that a `(language, script)` pair cannot describe. Docling refuses +them rather than quietly selecting a neighbouring recognizer: + +| Code | Names | +| ------------------------------------- | ---------------------------- | +| `jpn_vert`, `chi_sim_vert`, ... | Vertical-text models | +| `ita_old`, `spa_old`, `kat_old` | Historical orthographies | +| `equ` | Mathematical notation | + +Custom traineddata files you trained yourself fall in the same category and are not reachable +through `lang`. + +### When an engine has no model + +A language the selected engine cannot serve is an **error**, uniformly, for every engine. Docling +never quietly substitutes a different recognizer; the message names the languages that engine does +support, as canonical tags. Engines that run one language at a time (RapidOCR, Nemotron-OCR) take +the **first** tag and warn about the rest. The KServe client also sends only the first entry, but +it never raises for coverage: the deployment is the only thing that can. ## RapidOCR This section describes RapidOCR for versions `v3.9.1`, `v3.9.2`. +The engine's own vocabulary, in its own codes, is listed in +[Native OCR engines](OCR_native.md#rapidocr). + ### RapidOCR backends RapidOCR supports multiple backends. @@ -63,114 +219,70 @@ hr, hu, id, is, it, ku, la, lb, lt, lv, mi, ms, mt, nl, no, oc, pl, pt, qu, rm, rs_latin, sk, sl, sq, sv, sw, tl, tr, uz, vi, french, german ``` -Additionally the following aliases exist for PP-OCR v6: - -``` -zh -> ch -zh_cn -> ch -zh-cn -> ch -zh_tw -> chinese_cht -zh-tw -> chinese_cht -ja -> japan -jp -> japan -ko -> korean -``` - - Notices: -- German exists in 2 formats: `de`, `german`. -- French exists in 2 formats: `fr`, `french`. +- These are PP-OCR's own tokens, listed here to document what the checkpoints cover. You never + write them: docling takes BCP-47 tags and maps them onto these tokens for you. +- German exists in 2 formats: `de`, `german`; French in `fr`, `french`. Docling always picks the + two-letter one. - Korean is actually not supported in PP-OCR v6 (only the alias exists). -### RapidOCR language input semantic - -The following table explains the semantic of each language input for RapidOCR - -| Token | Meaning | -| ------------- | ------------------------------------------------------------------------------ | -| `af` | Afrikaans | -| `arabic` | Arabic-script family (9): Arabic, Persian, Uyghur, Urdu, Pashto, Kurdish, | -| | Sindhi, Baluchi, English | -| `az` | Azerbaijani | -| `bs` | Bosnian | -| `ca` | Catalan | -| `ch` | Chinese (Simplified) | -| `chinese_cht` | Chinese (Traditional) | -| `cs` | Czech | -| `cy` | Welsh | -| `cyrillic` | Cyrillic-script family (34): Russian, Belarusian, Ukrainian, Serbian | -| | (Cyrillic), Bulgarian, Mongolian, Abkhaz, Adyghe, Kabardian, Avar, Dargwa, | -| | Ingush, Chechen, Lak, Lezgian, Tabasaran, Kazakh, Kyrgyz, Tajik, Macedonian, | -| | Tatar, Chuvash, Bashkir, Meadow Mari, Moldovan, Udmurt, Komi, Ossetian, | -| | Buriat, Kalmyk, Tuvan, Yakut, Karakalpak, English | -| `da` | Danish | -| `de` | German | -| `devanagari` | Devanagari-script family (14): Hindi, Marathi, Nepali, Bihari, Maithili, | -| | Angika, Bhojpuri, Magahi, Sadri, Newari, Konkani (Goan), Sanskrit, Haryanvi, | -| | English | -| `el` | Greek | -| `en` | English | -| `es` | Spanish | -| `eslav` | East Slavic family, Cyrillic script (4): Russian, Belarusian, Ukrainian, | -| | English | -| `et` | Estonian | -| `eu` | Basque | -| `fi` | Finnish | -| `fr` | French | -| `ga` | Irish | -| `gl` | Galician | -| `hr` | Croatian | -| `hu` | Hungarian | -| `id` | Indonesian | -| `is` | Icelandic | -| `it` | Italian | -| `japan` | Japanese | -| `ka` | Kannada | -| `korean` | Korean | -| `ku` | Kurdish | -| `la` | Latin | -| `lb` | Luxembourgish | -| `lt` | Lithuanian | -| `lv` | Latvian | -| `mi` | Maori | -| `ms` | Malay | -| `mt` | Maltese | -| `nl` | Dutch | -| `no` | Norwegian | -| `oc` | Occitan | -| `pl` | Polish | -| `pt` | Portuguese | -| `qu` | Quechua | -| `rm` | Romansh | -| `ro` | Romanian | -| `rs_latin` | Serbian (Latin) | -| `sk` | Slovak | -| `sl` | Slovenian | -| `sq` | Albanian | -| `sv` | Swedish | -| `sw` | Swahili | -| `ta` | Tamil | -| `te` | Telugu | -| `th` | Thai | -| `tl` | Tagalog | -| `tr` | Turkish | -| `uz` | Uzbek | -| `vi` | Vietnamese | +### RapidOCR language input + +RapidOCR runs a **single** language per conversion. If `lang` holds more than one tag the first is +used and the rest are dropped with a warning. + +Tags resolve to a PP-OCR recognizer in this order: an explicit entry in the table below, then the +primary subtag if PP-OCR has it under that name, then the script family, then an error. + +| You write | PP-OCR token | Backbone | +| ------------------------------------------ | ----------------------- | ------------------ | +| `zh-Hans` / `zh-Hant` | `ch` / `chinese_cht` | v6 | +| `ja` | `japan` | v6 | +| `ko` | `korean` | v5 / v4 | +| `en`, `de`, `fr`, and the other v6 codes | the primary subtag | v6 | +| `sr-Latn` | `rs_latin` | v6 | +| `ru`, `uk`, `be` | `eslav` | v5 -- narrower | +| other Cyrillic-script languages | `cyrillic` | v5 / v4 | +| Arabic- and Devanagari-script languages | `arabic` / `devanagari` | v5 / v4 | +| `el`, `ta`, `te`, `th` | `el`, `ta`, `te`, `th` | v5 | +| `kn` | `ka` (PP-OCR's Kannada) | v4 | +| `ka-Geor` (Georgian) | -- | **error** | +| `latin`, `cyrillic`, `arabic`, | the token itself | v5 / v4 | +| `devanagari` (PP-OCR's own tokens) | | | +| an empty list | `ch` | the default | +| `mul` | -- | **error** | + +A language written in a script PP-OCR does not serve under that language's own name falls back to +the script family: `uz` is PP-OCR's Latin Uzbek, so `uz-Cyrl` resolves to `cyrillic` rather than +silently using the Latin recognizer. + +Prefetching follows the same vocabulary: + +```console +docling-tools models download rapidocr --rapidocr-backend-lang onnxruntime:th-Thai +``` ## EasyOCR This section describes EasyOCR for versions `v1.7.2`, `v1.7.1`. The model checkpoints are those of `gen2`. +The engine's own vocabulary, in its own codes, is listed in +[Native OCR engines](OCR_native.md#easyocr). + ### EasyOCR language support -EasyOCR accepts as input a list of languages. -The language resolution that takes place inside EasyOCR enables those models that can support all input languages. +EasyOCR accepts a list of languages and picks the recognition model that covers all of them. +Docling translates each BCP-47 tag into EasyOCR's own code -- `zh-Hant` becomes `ch_tra`, +`sr-Cyrl` becomes `rs_cyrillic`, `tg` becomes `tjk` -- and EasyOCR then selects the recognition +checkpoint for the script those codes share, so `ru` reaches the Cyrillic model without you +naming it. EasyOCR has no multilingual model, so `mul` raises; list the languages instead. The following table shows which recognition model is enabled per language combination -(the detection checkpoint `craft_mlt_25k.pth` is required in all cases): +(the detection checkpoint `craft_mlt_25k.pth` is required in all cases; the codes are EasyOCR's +own, shown here to explain the grouping): | Recognition checkpoint | Supported languages | | ---------------------- | ----------------------------------------------------------------------- | @@ -187,11 +299,11 @@ The following table shows which recognition model is enabled per language combin | `cyrillic_g2.pth` | `ru`, `rs_cyrillic`, `be`, `bg`, `uk`, `mn`, `abq`, `ady`, `kbd`, | | | `ava`, `dar`, `inh`, `che`, `lbe`, `lez`, `tab`, `tjk`, `en` | -Notice: keep the requested language list as short and specific as possible. Because the resolution -picks a model that covers *all* requested languages, adding a language you do not need downgrades the -model for the ones you do. For example, `["en"]` selects the English-specific `english_g2.pth`, while -`["en", "de"]` falls back to the broader `latin_g2.pth`, which is generally less accurate on English -text. +Notice: keep the requested language list as short and specific as possible. Because the +resolution picks a model that covers *all* requested languages, adding a language you do not need +downgrades the model for the ones you do. For example, `["en"]` selects the English-specific +`english_g2.pth`, while `["en", "de"]` falls back to the broader `latin_g2.pth`, which is generally +less accurate on English text. Check the semantic of easyocr language inputs here: https://www.jaided.ai/easyocr/ @@ -201,25 +313,45 @@ Check the semantic of easyocr language inputs here: https://www.jaided.ai/easyoc This section describes Nemotron-OCR for versions `v2.0.0`, `v2.0.2`. +The engine's own vocabulary, in its own codes, is listed in +[Native OCR engines](OCR_native.md#nemotron-ocr). + Nemotron works only on Linux and requires CUDA (Docling enforces 13.x). The following table shows the supported Python versions and languages -| Nemotron version | Python version | Supported language inputs | -| ---------------- | ---------------- | ------------------------------------------------------ | -| v2.0.0 | 3.12 only | `english` (alias `en`), `multilingual` (alias `multi`) | -| v2.0.2 | 3.11, 3.12, 3.13 | `english` (alias `en`), `multilingual` (alias `multi`) | - +| Nemotron version | Python version | Supported language inputs | +| ---------------- | ---------------- | ------------------------------------- | +| v2.0.0 | 3.12 only | `en`, `mul` | +| v2.0.2 | 3.11, 3.12, 3.13 | `en`, `mul` | -The "multi/multilingual" languages cover: English, Chinese (Simplified and Traditional), Japanese, Korean, and Russian +`en`, and an empty list, select the English recognizer. `mul` selects the multilingual one, as do +the languages it is trained on: English, Chinese (Simplified and Traditional), Japanese, Korean and +Russian. Any other language raises rather than silently loading the multilingual model -- ask for +`mul` explicitly if that is what you want. ## Tesseract - TesserOCR -Tesseract must be installed as a system package (see [installation](../getting_started/installation.md)). +Tesseract must be installed as a system package (see +[installation](../getting_started/installation.md)). TesserOCR is a python library that wraps the Tesseract engine. +The engine's own vocabulary, in its own codes, is listed in +[Native OCR engines](OCR_native.md#tesseract-tesserocr). + +Tesseract's own vocabulary *is* ISO 639-2/T, so most tags map straight through: `de` becomes `deu`, +`el` becomes `ell`, `cs` becomes `ces`. Docling handles the deviations for you -- `zh-Hant` becomes +`chi_tra`, `sr-Latn` becomes `srp_latn`, `az-Cyrl` becomes `aze_cyrl`, `ku` becomes `kmr`. A +`script/` traineddata file can be named directly, e.g. `lang=["script/Latin"]`. + +Languages are checked against the installed tessdata **at construction time**, so a missing +traineddata file now fails immediately with the installed set in the message, instead of failing +per page during conversion. + +An empty `lang` list runs Tesseract's per-page orientation and script detection. That requires +the `osd` traineddata; without it, `lang=[]` raises with an install hint. [Languages support](https://tesseract-ocr.github.io/tessdoc/Data-Files-in-different-versions.html) @@ -228,7 +360,16 @@ TesserOCR is a python library that wraps the Tesseract engine. This section describes ocrmac for versions `v1.0.0`, `v1.0.1`. +The engine's own vocabulary, in its own codes, is listed in +[Native OCR engines](OCR_native.md#ocrmac). + ocrmac is a thin wrapper around Apple's Vision framework. It is macOS-only and ships no model artifacts of its own — the recognizers are part of the operating system. The supported language set is therefore a property of the macOS version, not of the ocrmac release. +Vision's own vocabulary is BCP-47 with regions, so docling matches your tag against the list the +running macOS reports rather than mapping it through a table: `de` finds `de-DE`, `pt` finds +`pt-BR`, `zh-CN` finds `zh-Hans`. A tag with no close enough match raises, and the message lists +what this particular macOS actually offers. An empty `lang` list hands the choice to Vision's own +automatic behaviour; `mul` is not supported. + diff --git a/docs/concepts/OCR_native.md b/docs/concepts/OCR_native.md new file mode 100644 index 0000000000..f8fc64bc71 --- /dev/null +++ b/docs/concepts/OCR_native.md @@ -0,0 +1,234 @@ +# Native OCR engines + +## Overview + +Docling supports multiple OCR engines that can be installed as extra packages: + +- [RapidOCR](https://github.com/RapidAI/RapidOCR) +- [Nemotron-OCR](https://huggingface.co/nvidia/nemotron-ocr-v2) +- [EasyOCR](https://github.com/jaidedai/easyocr) +- [ocrmac](https://github.com/straussmaximilian/ocrmac) +- [tesseract-CLI](https://github.com/tesseract-ocr/tesseract) +- [tesserocr](https://github.com/sirfz/tesserocr) + + +## RapidOCR + +This section describes RapidOCR for versions `v3.9.1`, `v3.9.2`. + +### RapidOCR backends + +RapidOCR supports multiple backends. +Docling currently (2026.07.28) supports: "onnxruntime" (default), "openvino", "paddle", "torch". + +RapidOCR relies on the [PP-OCR](https://rapidai.github.io/RapidOCRDocs/main/model_list/#_2) models. +Docling currently (2026.07.28) supports: "PP-OCR v4", "PP-OCR v5", "PP-OCR v6". + +**PP-OCR versions supported by each rapidocr backend:** + +| Backend | PP-OCR versions | +| ----------- | -------------------- | +| onnxruntime | v4, v5, v6 | +| openvino | v4, v5, v6 | +| paddle | v4, v5, v6 | +| torch | v4, v5 (ch only), v6 | + +Notice: torch on PP-OCRv5 supports ONLY chinese. + + +### RapidOCR language support + +**PP-OCRv4 supported languages/scripts:** + +``` +arabic, ch, chinese_cht, cyrillic, devanagari, en, japan, ka, korean, latin, ta, te +``` + +Notice: `cyrillic`, `devanagari`, `latin` are actually scripts and each one supports multiple +languages. + + +**PP-OCRv5 supported languages/scripts:** + +``` +arabic, ch, cyrillic, devanagari, el, en, eslav, korean, latin, ta, te, th +``` + + +**PP-OCRv6 supported languages:** + +``` +ch, chinese_cht, en, japan, af, az, bs, ca, cs, cy, da, de, es, et, eu, fi, fr, ga, gl, +hr, hu, id, is, it, ku, la, lb, lt, lv, mi, ms, mt, nl, no, oc, pl, pt, qu, rm, ro, +rs_latin, sk, sl, sq, sv, sw, tl, tr, uz, vi, french, german +``` + +Additionally the following aliases exist for PP-OCR v6: + +``` +zh -> ch +zh_cn -> ch +zh-cn -> ch +zh_tw -> chinese_cht +zh-tw -> chinese_cht +ja -> japan +jp -> japan +ko -> korean +``` + + +Notices: + +- German exists in 2 formats: `de`, `german`. +- French exists in 2 formats: `fr`, `french`. +- Korean is actually not supported in PP-OCR v6 (only the alias exists). + + +### RapidOCR language input semantic + +The following table explains the semantic of each language input for RapidOCR + +| Token | Meaning | +| ------------- | ------------------------------------------------------------------------------ | +| `af` | Afrikaans | +| `arabic` | Arabic-script family (9): Arabic, Persian, Uyghur, Urdu, Pashto, Kurdish, | +| | Sindhi, Baluchi, English | +| `az` | Azerbaijani | +| `bs` | Bosnian | +| `ca` | Catalan | +| `ch` | Chinese (Simplified) | +| `chinese_cht` | Chinese (Traditional) | +| `cs` | Czech | +| `cy` | Welsh | +| `cyrillic` | Cyrillic-script family (34): Russian, Belarusian, Ukrainian, Serbian | +| | (Cyrillic), Bulgarian, Mongolian, Abkhaz, Adyghe, Kabardian, Avar, Dargwa, | +| | Ingush, Chechen, Lak, Lezgian, Tabasaran, Kazakh, Kyrgyz, Tajik, Macedonian, | +| | Tatar, Chuvash, Bashkir, Meadow Mari, Moldovan, Udmurt, Komi, Ossetian, | +| | Buriat, Kalmyk, Tuvan, Yakut, Karakalpak, English | +| `da` | Danish | +| `de` | German | +| `devanagari` | Devanagari-script family (14): Hindi, Marathi, Nepali, Bihari, Maithili, | +| | Angika, Bhojpuri, Magahi, Sadri, Newari, Konkani (Goan), Sanskrit, Haryanvi, | +| | English | +| `el` | Greek | +| `en` | English | +| `es` | Spanish | +| `eslav` | East Slavic family, Cyrillic script (4): Russian, Belarusian, Ukrainian, | +| | English | +| `et` | Estonian | +| `eu` | Basque | +| `fi` | Finnish | +| `fr` | French | +| `ga` | Irish | +| `gl` | Galician | +| `hr` | Croatian | +| `hu` | Hungarian | +| `id` | Indonesian | +| `is` | Icelandic | +| `it` | Italian | +| `japan` | Japanese | +| `ka` | Kannada | +| `korean` | Korean | +| `ku` | Kurdish | +| `la` | Latin | +| `lb` | Luxembourgish | +| `lt` | Lithuanian | +| `lv` | Latvian | +| `mi` | Maori | +| `ms` | Malay | +| `mt` | Maltese | +| `nl` | Dutch | +| `no` | Norwegian | +| `oc` | Occitan | +| `pl` | Polish | +| `pt` | Portuguese | +| `qu` | Quechua | +| `rm` | Romansh | +| `ro` | Romanian | +| `rs_latin` | Serbian (Latin) | +| `sk` | Slovak | +| `sl` | Slovenian | +| `sq` | Albanian | +| `sv` | Swedish | +| `sw` | Swahili | +| `ta` | Tamil | +| `te` | Telugu | +| `th` | Thai | +| `tl` | Tagalog | +| `tr` | Turkish | +| `uz` | Uzbek | +| `vi` | Vietnamese | + +## EasyOCR + +This section describes EasyOCR for versions `v1.7.2`, `v1.7.1`. +The model checkpoints are those of `gen2`. + +### EasyOCR language support + +EasyOCR accepts as input a list of languages. +The language resolution that takes place inside EasyOCR enables those models that can support all input languages. + +The following table shows which recognition model is enabled per language combination +(the detection checkpoint `craft_mlt_25k.pth` is required in all cases): + +| Recognition checkpoint | Supported languages | +| ---------------------- | ----------------------------------------------------------------------- | +| `english_g2.pth` | `en` | +| `latin_g2.pth` | `af`, `az`, `bs`, `cs`, `cy`, `da`, `de`, `en`, `es`, `et`, `fr`, `ga`, | +| | `hr`, `hu`, `id`, `is`, `it`, `ku`, `la`, `lt`, `lv`, `mi`, `ms`, `mt`, | +| | `nl`, `no`, `oc`, `pi`, `pl`, `pt`, `ro`, `rs_latin`, `sk`, `sl`, `sq`, | +| | `sv`, `sw`, `tl`, `tr`, `uz`, `vi` | +| `zh_sim_g2.pth` | `ch_sim` + `en` | +| `japanese_g2.pth` | `ja` + `en` | +| `korean_g2.pth` | `ko` + `en` | +| `telugu.pth` | `te` + `en` | +| `kannada.pth` | `kn` + `en` | +| `cyrillic_g2.pth` | `ru`, `rs_cyrillic`, `be`, `bg`, `uk`, `mn`, `abq`, `ady`, `kbd`, | +| | `ava`, `dar`, `inh`, `che`, `lbe`, `lez`, `tab`, `tjk`, `en` | + +Notice: keep the requested language list as short and specific as possible. Because the resolution +picks a model that covers *all* requested languages, adding a language you do not need downgrades the +model for the ones you do. For example, `["en"]` selects the English-specific `english_g2.pth`, while +`["en", "de"]` falls back to the broader `latin_g2.pth`, which is generally less accurate on English +text. + +Check the semantic of easyocr language inputs here: https://www.jaided.ai/easyocr/ + + + +## Nemotron-OCR + +This section describes Nemotron-OCR for versions `v2.0.0`, `v2.0.2`. + +Nemotron works only on Linux and requires CUDA (Docling enforces 13.x). + +The following table shows the supported Python versions and languages + + +| Nemotron version | Python version | Supported language inputs | +| ---------------- | ---------------- | ------------------------------------------------------ | +| v2.0.0 | 3.12 only | `english` (alias `en`), `multilingual` (alias `multi`) | +| v2.0.2 | 3.11, 3.12, 3.13 | `english` (alias `en`), `multilingual` (alias `multi`) | + + +The "multi/multilingual" languages cover: English, Chinese (Simplified and Traditional), Japanese, Korean, and Russian + + +## Tesseract - TesserOCR + +Tesseract must be installed as a system package (see [installation](../getting_started/installation.md)). +TesserOCR is a python library that wraps the Tesseract engine. + + +[Languages support](https://tesseract-ocr.github.io/tessdoc/Data-Files-in-different-versions.html) + + +## OcrMac + +This section describes ocrmac for versions `v1.0.0`, `v1.0.1`. + +ocrmac is a thin wrapper around Apple's Vision framework. It is macOS-only and ships no model +artifacts of its own — the recognizers are part of the operating system. The supported language set +is therefore a property of the macOS version, not of the ocrmac release. + diff --git a/docs/concepts/plugins.md b/docs/concepts/plugins.md index 6e44926066..d2da006024 100644 --- a/docs/concepts/plugins.md +++ b/docs/concepts/plugins.md @@ -64,6 +64,48 @@ def ocr_engines(): where `YourOcrModel` must implement the [`BaseOcrModel`](https://github.com/docling-project/docling/blob/main/docling/models/base_ocr_model.py#L40) and provide an options class derived from [`OcrOptions`](https://github.com/docling-project/docling/blob/main/docling/datamodel/pipeline_options.py#L184). +#### OCR languages in an external engine + +`OcrOptions.lang` is validated and canonicalized by the base class: your engine receives BCP-47 +tags such as `en-Latn` and `zh-Hans`, never `en` or `chinese`. See +[OCR engines](OCR.md#language-selection) for the user-facing contract. + +With no further work, `BaseOcrModel.map_ocr_language` hands your engine the **primary subtag** +(`en`, `zh`), which is what most ISO-639-based engines want. Two things still need attention: +restate your options default in BCP-47, and note that the primary subtag alone loses the +Simplified/Traditional distinction. + +To participate fully, override three members: + +```py +from docling.exceptions import OcrLanguageNotSupportedError +from docling.models.base_ocr_model import BaseOcrModel +from docling.utils.ocr_language import OcrLanguage, OcrLanguageSupport + + +class YourOcrModel(BaseOcrModel): + # What the engine can do with a language request. + language_support = OcrLanguageSupport( + multiple_languages=False, # True if several languages can run at once + ) + + def supported_ocr_languages(self) -> list[str]: + # Canonical tags this instance can serve, for error messages. + return ["en-Latn", "de-Latn"] + + def map_ocr_language(self, language: OcrLanguage) -> str | list[str]: + # Map one canonical tag onto your engine's native code(s). + if language.tag not in self.supported_ocr_languages(): + raise OcrLanguageNotSupportedError(type(self).__name__, language.tag) + return language.bcp47_language +``` + +`BaseOcrModel.resolve_ocr_languages()` then drops every language after the first on a +single-language engine, with a warning naming what it kept. It does not touch the error path: an +`OcrLanguageNotSupportedError` your `map_ocr_language` raises propagates unchanged, which is why +the sample above attaches `supported=` itself. Call it once from `__init__`, inside your +`if self.enabled:` block, and use the result in place of `options.lang`. + ### Layout engine factory The layout engine factory allows to provide more layout engines to the Docling users. diff --git a/docs/examples/custom_convert.py b/docs/examples/custom_convert.py index 112cf2658a..b5721d07e7 100644 --- a/docs/examples/custom_convert.py +++ b/docs/examples/custom_convert.py @@ -28,7 +28,8 @@ # - If you don't have the test data, update `input_doc_path` to a local PDF. # # Notes -# - EasyOCR language: adjust `pipeline_options.ocr_options.lang` (e.g., ["en"], ["es"], ["en", "de"]). +# - EasyOCR language: adjust `pipeline_options.ocr_options.lang` with BCP-47 tags +# (e.g., ["en"], ["es"], ["en", "de"], ["zh-Hant"]). # - Accelerators: tune `AcceleratorOptions` to select CPU/GPU or threads. # - Exports: JSON, plain text, Markdown, and doctags are saved in `scratch/`. diff --git a/docs/examples/tesseract_lang_detection.py b/docs/examples/tesseract_lang_detection.py index d11641cfe8..73ec34e002 100644 --- a/docs/examples/tesseract_lang_detection.py +++ b/docs/examples/tesseract_lang_detection.py @@ -2,7 +2,7 @@ # Detect language automatically with Tesseract OCR and force full-page OCR. # # What this example does -# - Configures Tesseract (CLI in this snippet) with `lang=["auto"]`. +# - Configures Tesseract (CLI in this snippet) with an empty `lang` list. # - Forces full-page OCR and prints the recognized text as Markdown. # # How to run @@ -12,8 +12,10 @@ # Notes # - You can switch to `TesseractOcrOptions` instead of `TesseractCliOcrOptions`. # - Language packs must be installed; set `TESSDATA_PREFIX` if Tesseract -# cannot find language data. Using `lang=["auto"]` requires traineddata -# that supports script/language detection on your system. +# cannot find language data. An empty `lang` list means "let the engine +# decide": for Tesseract that is per-page orientation and script detection, +# which requires the `osd` traineddata plus the per-script files it may +# detect. # %% @@ -39,9 +41,9 @@ def main(): data_folder = Path(__file__).parent / "../../tests/data" input_doc_path = data_folder / "pdf/sources/2206.01062.pdf" - # Set lang=["auto"] with a tesseract OCR engine: TesseractOcrOptions, TesseractCliOcrOptions - # ocr_options = TesseractOcrOptions(lang=["auto"], mode=OcrMode.FULL_PAGE) - ocr_options = TesseractCliOcrOptions(lang=["auto"], mode=OcrMode.FULL_PAGE) + # Set lang=[] with a tesseract OCR engine: TesseractOcrOptions, TesseractCliOcrOptions + # ocr_options = TesseractOcrOptions(lang=[], mode=OcrMode.FULL_PAGE) + ocr_options = TesseractCliOcrOptions(lang=[], mode=OcrMode.FULL_PAGE) pipeline_options = PdfPipelineOptions(do_ocr=True, ocr_options=ocr_options) diff --git a/docs/faq/index.md b/docs/faq/index.md index c620d3eeeb..08a115f569 100644 --- a/docs/faq/index.md +++ b/docs/faq/index.md @@ -170,8 +170,12 @@ This is a collection of FAQ collected from the user questions on :' (e.g. 'onnxruntime:el', 'torch:korean'). Repeat for multiple. Replaces the default set. | +| `--easyocr-lang` | `text` (repeatable) | | OCR language to prefetch for EasyOCR, as a BCP-47 tag (e.g. 'de', 'zh-Hant', 'ru'). Repeat for multiple. | +| `--rapidocr-backend-lang` | `text` (repeatable) | | RapidOCR checkpoint set to prefetch, as ':' with a BCP-47 language (e.g. 'onnxruntime:el', 'torch:ko'). Repeat for multiple. Replaces the default set. | #### `docling-tools models download-hf-repo` diff --git a/docs/usage/advanced_options.md b/docs/usage/advanced_options.md index dcb910476b..f4f027549a 100644 --- a/docs/usage/advanced_options.md +++ b/docs/usage/advanced_options.md @@ -14,18 +14,16 @@ Downloading layout model... Downloading tableformer model... Downloading picture classifier model... Downloading code formula model... -Downloading rapidocr torch chinese models... -Downloading rapidocr torch english models... -Downloading rapidocr onnxruntime chinese models... -Downloading rapidocr onnxruntime english models... +Downloading rapidocr torch zh-Hans models... +Downloading rapidocr onnxruntime zh-Hans models... Models downloaded into $HOME/.cache/docling/models. ``` To prefetch EasyOCR recognition models for specific languages, repeat -`--easyocr-lang` with the same language codes used by `EasyOcrOptions.lang`: +`--easyocr-lang` with the same BCP-47 tags used by `EasyOcrOptions.lang`: ```sh -$ docling-tools models download easyocr --easyocr-lang ch_sim --easyocr-lang ja +$ docling-tools models download easyocr --easyocr-lang zh-Hans --easyocr-lang ja ``` Alternatively, models can be programmatically downloaded using `docling.utils.model_downloader.download_models()`. diff --git a/docs/usage/model_catalog.md b/docs/usage/model_catalog.md index 1d2303ddc8..d4b62d56c5 100644 --- a/docs/usage/model_catalog.md +++ b/docs/usage/model_catalog.md @@ -225,6 +225,9 @@ object-detection path — but selecting `DOCLING_LAYOUT_V2` warns and falls back ### OCR Engines +Languages are given as BCP-47 tags for every engine; see +[OCR engines](../concepts/OCR.md#language-selection). + | OCR Engine | Backend | Language Support | Notes | |------------|---------|------------------|-------| | Tesseract | CLI or tesserocr | 100+ languages | Most widely used, good accuracy | @@ -316,8 +319,9 @@ classifier_options = DocumentPictureClassifierOptions.from_preset("document_figu ```python from docling.datamodel.pipeline_options import TesseractOcrOptions -# Use Tesseract with English and German -ocr_options = TesseractOcrOptions(lang=["eng", "deu"]) +# Use Tesseract with English and German. Languages are BCP-47 tags; +# `eng`/`deu` still work and canonicalize to `en-Latn`/`de-Latn`. +ocr_options = TesseractOcrOptions(lang=["en", "de"]) ``` ### VLM Convert (Full Page) diff --git a/mkdocs.yml b/mkdocs.yml index c921f75227..b0118b26de 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -92,6 +92,7 @@ nav: - Chunking: concepts/chunking.md - Plugins: concepts/plugins.md - OCR engines: concepts/OCR.md + - Native OCR engines: concepts/OCR_native.md - Examples: # NOTE: notebook (.ipynb) and jupytext percent-format (.py) sources under # docs/examples/ are pre-rendered to markdown by scripts/render_notebooks.py diff --git a/pyproject.toml b/pyproject.toml index 661d405952..8cc3988200 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ dependencies = [ 'certifi>=2024.7.4', 'pluggy>=1.0.0,<2.0.0', 'tqdm>=4.65.0,<5.0.0', + "langcodes>=3.5.0,<4.0.0", ] [project.urls] diff --git a/tach.toml b/tach.toml index 7d35fb37cd..9797129f35 100644 --- a/tach.toml +++ b/tach.toml @@ -32,6 +32,11 @@ path = "docling.chunking" layer = "foundation" depends_on = [] +[[modules]] +path = "docling.utils.ocr_language" +layer = "foundation" +depends_on = [] + [[modules]] path = "docling.backend" layer = "core" @@ -91,6 +96,7 @@ depends_on = [ "docling.models.inference_engines.vlm", "docling.models.utils", "docling.utils", + "docling.utils.ocr_language", ] [[modules]] @@ -190,6 +196,8 @@ layer = "core" depends_on = [ "docling.datamodel", "docling.datamodel.pipeline_options_vlm_model", + "docling.exceptions", + "docling.utils.ocr_language", ] [[modules]] @@ -298,9 +306,11 @@ path = "docling.models.stages.ocr" layer = "core" depends_on = [ "docling.datamodel", + "docling.exceptions", "docling.models", "docling.models.inference_engines.common", "docling.utils", + "docling.utils.ocr_language", ] [[modules]] diff --git a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test.ocrmac.pdf_aware_layout_regions.doctags.txt b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test.ocrmac.pdf_aware_layout_regions.doctags.txt index 08317e33df..5682a134dd 100644 --- a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test.ocrmac.pdf_aware_layout_regions.doctags.txt +++ b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test.ocrmac.pdf_aware_layout_regions.doctags.txt @@ -1,4 +1,2 @@ -Docling bundles PDF document conversion to -JSON and Markdown in an easy self contained -package +Docling bundles PDF document conversion to JSON and Markdown in an easy self contained package \ No newline at end of file diff --git a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test.ocrmac.pdf_aware_layout_regions.json b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test.ocrmac.pdf_aware_layout_regions.json index da918c258f..766f2d1248 100644 --- a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test.ocrmac.pdf_aware_layout_regions.json +++ b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test.ocrmac.pdf_aware_layout_regions.json @@ -19,12 +19,6 @@ "children": [ { "$ref": "#/texts/0" - }, - { - "$ref": "#/texts/1" - }, - { - "$ref": "#/texts/2" } ], "content_layer": "body", @@ -45,74 +39,20 @@ { "page_no": 1, "bbox": { - "l": 9.4, - "t": 828.97, - "r": 441.6, - "b": 806.58, - "coord_origin": "BOTTOMLEFT" - }, - "charspan": [ - 0, - 42 - ] - } - ], - "orig": "Docling bundles PDF document conversion to", - "text": "Docling bundles PDF document conversion to" - }, - { - "self_ref": "#/texts/1", - "parent": { - "$ref": "#/body" - }, - "children": [], - "content_layer": "body", - "label": "text", - "prov": [ - { - "page_no": 1, - "bbox": { - "l": 7.33, - "t": 800.74, - "r": 441.01, - "b": 782.14, - "coord_origin": "BOTTOMLEFT" - }, - "charspan": [ - 0, - 43 - ] - } - ], - "orig": "JSON and Markdown in an easy self contained", - "text": "JSON and Markdown in an easy self contained" - }, - { - "self_ref": "#/texts/2", - "parent": { - "$ref": "#/body" - }, - "children": [], - "content_layer": "body", - "label": "text", - "prov": [ - { - "page_no": 1, - "bbox": { - "l": 9.99, - "t": 774.16, - "r": 89.27, - "b": 754.9, + "l": 70.33, + "t": 764.97, + "r": 504.6, + "b": 690.9, "coord_origin": "BOTTOMLEFT" }, "charspan": [ 0, - 7 + 94 ] } ], - "orig": "package", - "text": "package" + "orig": "Docling bundles PDF document conversion to JSON and Markdown in an easy self contained package", + "text": "Docling bundles PDF document conversion to JSON and Markdown in an easy self contained package" } ], "pictures": [], diff --git a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test.ocrmac.pdf_aware_layout_regions.md b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test.ocrmac.pdf_aware_layout_regions.md index 2a64c453dd..428965464a 100644 --- a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test.ocrmac.pdf_aware_layout_regions.md +++ b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test.ocrmac.pdf_aware_layout_regions.md @@ -1,5 +1 @@ -Docling bundles PDF document conversion to - -JSON and Markdown in an easy self contained - -package \ No newline at end of file +Docling bundles PDF document conversion to JSON and Markdown in an easy self contained package \ No newline at end of file diff --git a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_180.ocrmac.pdf_aware_layout_regions.doctags.txt b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_180.ocrmac.pdf_aware_layout_regions.doctags.txt index 3b34ecd167..0cc59aefdb 100644 --- a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_180.ocrmac.pdf_aware_layout_regions.doctags.txt +++ b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_180.ocrmac.pdf_aware_layout_regions.doctags.txt @@ -1,4 +1,2 @@ -package -JSON and Markdown in an easy self contained -Docling bundles PDF document conversion to +Docling bundles PDF document conversion to JSON and Markdown in an easy self contained package \ No newline at end of file diff --git a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_180.ocrmac.pdf_aware_layout_regions.json b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_180.ocrmac.pdf_aware_layout_regions.json index 043c9a2380..7c2ba615df 100644 --- a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_180.ocrmac.pdf_aware_layout_regions.json +++ b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_180.ocrmac.pdf_aware_layout_regions.json @@ -19,12 +19,6 @@ "children": [ { "$ref": "#/texts/0" - }, - { - "$ref": "#/texts/1" - }, - { - "$ref": "#/texts/2" } ], "content_layer": "body", @@ -45,74 +39,20 @@ { "page_no": 1, "bbox": { - "l": 361.07, - "t": 827.92, - "r": 439.68, - "b": 809.92, - "coord_origin": "BOTTOMLEFT" - }, - "charspan": [ - 0, - 7 - ] - } - ], - "orig": "package", - "text": "package" - }, - { - "self_ref": "#/texts/1", - "parent": { - "$ref": "#/body" - }, - "children": [], - "content_layer": "body", - "label": "text", - "prov": [ - { - "page_no": 1, - "bbox": { - "l": 5.64, - "t": 802.49, - "r": 443.48, - "b": 779.96, - "coord_origin": "BOTTOMLEFT" - }, - "charspan": [ - 0, - 43 - ] - } - ], - "orig": "JSON and Markdown in an easy self contained", - "text": "JSON and Markdown in an easy self contained" - }, - { - "self_ref": "#/texts/2", - "parent": { - "$ref": "#/body" - }, - "children": [], - "content_layer": "body", - "label": "text", - "prov": [ - { - "page_no": 1, - "bbox": { - "l": 8.66, - "t": 774.59, - "r": 439.01, - "b": 755.92, + "l": 87.64, + "t": 149.92, + "r": 525.48, + "b": 77.92, "coord_origin": "BOTTOMLEFT" }, "charspan": [ 0, - 42 + 94 ] } ], - "orig": "Docling bundles PDF document conversion to", - "text": "Docling bundles PDF document conversion to" + "orig": "Docling bundles PDF document conversion to JSON and Markdown in an easy self contained package", + "text": "Docling bundles PDF document conversion to JSON and Markdown in an easy self contained package" } ], "pictures": [], diff --git a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_180.ocrmac.pdf_aware_layout_regions.md b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_180.ocrmac.pdf_aware_layout_regions.md index 120ab1cc59..428965464a 100644 --- a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_180.ocrmac.pdf_aware_layout_regions.md +++ b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_180.ocrmac.pdf_aware_layout_regions.md @@ -1,5 +1 @@ -package - -JSON and Markdown in an easy self contained - -Docling bundles PDF document conversion to \ No newline at end of file +Docling bundles PDF document conversion to JSON and Markdown in an easy self contained package \ No newline at end of file diff --git a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_270.ocrmac.pdf_aware_layout_regions.doctags.txt b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_270.ocrmac.pdf_aware_layout_regions.doctags.txt index c1d8947f19..47bba067cc 100644 --- a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_270.ocrmac.pdf_aware_layout_regions.doctags.txt +++ b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_270.ocrmac.pdf_aware_layout_regions.doctags.txt @@ -1,2 +1,2 @@ -package JSON and Markdown in an easy self contained Docling bundles PDF document conversion to +Docling bundles PDF document conversion to JSON and Markdown in an easy self contained package \ No newline at end of file diff --git a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_270.ocrmac.pdf_aware_layout_regions.json b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_270.ocrmac.pdf_aware_layout_regions.json index d659e60805..465851909b 100644 --- a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_270.ocrmac.pdf_aware_layout_regions.json +++ b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_270.ocrmac.pdf_aware_layout_regions.json @@ -39,48 +39,20 @@ { "page_no": 1, "bbox": { - "l": 10.67, - "t": 583.87, - "r": 29.33, - "b": 505.2, + "l": 691.67, + "t": 523.87, + "r": 763.67, + "b": 90.86, "coord_origin": "BOTTOMLEFT" }, "charspan": [ 0, - 7 - ] - }, - { - "page_no": 1, - "bbox": { - "l": 37.33, - "t": 585.87, - "r": 56.0, - "b": 152.53, - "coord_origin": "BOTTOMLEFT" - }, - "charspan": [ - 8, - 51 - ] - }, - { - "page_no": 1, - "bbox": { - "l": 64.0, - "t": 582.53, - "r": 82.67, - "b": 152.53, - "coord_origin": "BOTTOMLEFT" - }, - "charspan": [ - 52, 94 ] } ], - "orig": "package JSON and Markdown in an easy self contained Docling bundles PDF document conversion to", - "text": "package JSON and Markdown in an easy self contained Docling bundles PDF document conversion to" + "orig": "Docling bundles PDF document conversion to JSON and Markdown in an easy self contained package", + "text": "Docling bundles PDF document conversion to JSON and Markdown in an easy self contained package" } ], "pictures": [], diff --git a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_270.ocrmac.pdf_aware_layout_regions.md b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_270.ocrmac.pdf_aware_layout_regions.md index 957d4d98c2..428965464a 100644 --- a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_270.ocrmac.pdf_aware_layout_regions.md +++ b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_270.ocrmac.pdf_aware_layout_regions.md @@ -1 +1 @@ -package JSON and Markdown in an easy self contained Docling bundles PDF document conversion to \ No newline at end of file +Docling bundles PDF document conversion to JSON and Markdown in an easy self contained package \ No newline at end of file diff --git a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_90.ocrmac.pdf_aware_layout_regions.doctags.txt b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_90.ocrmac.pdf_aware_layout_regions.doctags.txt index ee1cdd4e5d..cc49d10cb9 100644 --- a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_90.ocrmac.pdf_aware_layout_regions.doctags.txt +++ b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_90.ocrmac.pdf_aware_layout_regions.doctags.txt @@ -1,2 +1,2 @@ -Docling bundles PDF document conversion to JSON and Markdown in an easy self contained package +Docling bundles PDF document conversion to JSON and Markdown in an easy self contained package \ No newline at end of file diff --git a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_90.ocrmac.pdf_aware_layout_regions.json b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_90.ocrmac.pdf_aware_layout_regions.json index 96bd05efdb..800c40dc10 100644 --- a/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_90.ocrmac.pdf_aware_layout_regions.json +++ b/tests/data/ocr/groundtruth/ocrmac/pdf_aware_layout_regions/ocr_test_rotated_90.ocrmac.pdf_aware_layout_regions.json @@ -39,38 +39,24 @@ { "page_no": 1, "bbox": { - "l": 7.52, - "t": 585.81, - "r": 30.08, - "b": 153.6, + "l": 74.52, + "t": 503.87, + "r": 123.0, + "b": 70.86, "coord_origin": "BOTTOMLEFT" }, "charspan": [ 0, - 42 - ] - }, - { - "page_no": 1, - "bbox": { - "l": 37.33, - "t": 585.87, - "r": 56.0, - "b": 152.86, - "coord_origin": "BOTTOMLEFT" - }, - "charspan": [ - 43, 86 ] }, { "page_no": 1, "bbox": { - "l": 64.15, - "t": 234.29, - "r": 84.22, - "b": 154.75, + "l": 131.15, + "t": 152.29, + "r": 151.22, + "b": 72.75, "coord_origin": "BOTTOMLEFT" }, "charspan": [ diff --git a/tests/test_backend_webp.py b/tests/test_backend_webp.py index 0a2e064301..dd0b46bd65 100644 --- a/tests/test_backend_webp.py +++ b/tests/test_backend_webp.py @@ -58,9 +58,9 @@ def test_e2e_webp_conversions(): TesseractCliOcrOptions(), EasyOcrOptions(mode=OcrMode.FULL_PAGE), TesseractOcrOptions(mode=OcrMode.FULL_PAGE), - TesseractOcrOptions(mode=OcrMode.FULL_PAGE, lang=["auto"]), + TesseractOcrOptions(mode=OcrMode.FULL_PAGE, lang=[]), TesseractCliOcrOptions(mode=OcrMode.FULL_PAGE), - TesseractCliOcrOptions(mode=OcrMode.FULL_PAGE, lang=["auto"]), + TesseractCliOcrOptions(mode=OcrMode.FULL_PAGE, lang=[]), ] # rapidocr is only available for Python >=3.6,<3.14 diff --git a/tests/test_cli.py b/tests/test_cli.py index 3d5491e95c..423b927fc7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -585,6 +585,8 @@ def test_image_export_policy_covers_all_output_formats(): def test_split_list_handles_none_and_delimiters(): assert _split_list(None) is None assert _split_list("a,b;c") == ["a", "b", "c"] + # Values are stripped and blanks dropped, so `--ocr-lang "en, de"` works. + assert _split_list("a, b ;c,") == ["a", "b", "c"] def test_cli_audio_auto_detection(tmp_path): diff --git a/tests/test_cli_remote.py b/tests/test_cli_remote.py index 3d27ed9cba..06dcfe3c02 100644 --- a/tests/test_cli_remote.py +++ b/tests/test_cli_remote.py @@ -6,7 +6,6 @@ The service client is faked so these run without a live docling-serve instance. """ -import re from pathlib import Path, PurePath import pytest @@ -16,13 +15,10 @@ from docling.cli.remote import _parse_page_range from docling.datamodel.base_models import ConversionStatus, InputFormat, OutputFormat -runner = CliRunner() +# Under CI Rich thinks it has a terminal and colours the help screen. +# `TERM=dumb` turns that off, so the text can be matched as it is written. +runner = CliRunner(env={"TERM": "dumb"}) pytestmark = pytest.mark.external_service -_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") - - -def _strip_ansi(text: str) -> str: - return _ANSI_RE.sub("", text) class _FakeDoc: @@ -97,7 +93,7 @@ def _patch_client(monkeypatch): def test_remote_help_is_self_sufficient(): result = runner.invoke(app, ["convert-remote", "--help"]) assert result.exit_code == 0 - help_text = _strip_ansi(result.output) + help_text = result.output for marker in ( "Authentication", "Exit codes", @@ -196,13 +192,42 @@ def test_remote_maps_conversion_options(tmp_path, _patch_client): assert opts.to_formats == [OutputFormat.MARKDOWN, OutputFormat.JSON] assert opts.do_ocr is False assert opts.do_table_structure is False - assert opts.ocr_lang == ["en", "de"] + # `en,de` still works; it is canonicalized rather than rejected. + assert opts.ocr_lang == ["en-Latn", "de-Latn"] assert opts.page_range == (2, 5) # Both requested formats are written locally. assert (output / "report.md").exists() assert (output / "report.json").exists() +def test_remote_empty_ocr_lang_asks_the_engine_to_choose(tmp_path, _patch_client): + """`--ocr-lang ""` is not the same request as omitting the option. + + Omitting it leaves the service on its default languages; the empty value is + how `convert-remote` asks the engine to choose, the way local `convert` does. + """ + source = tmp_path / "report.pdf" + source.write_bytes(b"%PDF-1.4") + output = tmp_path / "out" + + result = runner.invoke( + app, + [ + "convert-remote", + str(source), + "--service-url", + "https://docling.example.com", + "--ocr-lang", + "", + "--output", + str(output), + ], + ) + + assert result.exit_code == 0, result.output + assert _FakeClient.instances[-1].captured_options.ocr_lang == [] + + def test_remote_credentials_from_env(tmp_path, monkeypatch, _patch_client): source = tmp_path / "report.pdf" source.write_bytes(b"%PDF-1.4") diff --git a/tests/test_cli_tools.py b/tests/test_cli_tools.py index af96796987..1efdc4d3e1 100644 --- a/tests/test_cli_tools.py +++ b/tests/test_cli_tools.py @@ -21,7 +21,10 @@ from docling.cli.models import _AvailableModels, _default_models from docling.cli.tools import app -runner = CliRunner() +# Under CI Rich thinks it has a terminal and styles the error panel, landing +# escapes between the border and the wrapped halves of a sentence. +# `TERM=dumb` turns that off, so the panel arrives as plain wrapped text. +runner = CliRunner(env={"TERM": "dumb"}) @pytest.fixture @@ -55,7 +58,6 @@ def _enabled(recorded: dict[str, Any]) -> set[str]: return {key for key, value in recorded.items() if key.startswith("with_") and value} -_ANSI_ESCAPE = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") _BOX_DRAWING = re.compile(r"[\u2500-\u257f]") @@ -63,15 +65,9 @@ def _flat(output: str) -> str: """Reduce a Rich error panel to a single line of plain text. Typer renders ``BadParameter`` messages inside a bordered panel and hard - wraps them, so error text cannot be matched against the raw output. When - the output stream is a terminal -- as it is under CI -- Rich also emits - colour codes, including between the wrapped halves of a sentence, so the - escapes have to go before whitespace is collapsed or the message is still - split. + wraps them, so error text cannot be matched against the raw output. """ - stripped = _ANSI_ESCAPE.sub("", output) - stripped = _BOX_DRAWING.sub(" ", stripped) - return re.sub(r"\s+", " ", stripped) + return re.sub(r"\s+", " ", _BOX_DRAWING.sub(" ", output)) def test_tools_help_lists_models_subcommand(): diff --git a/tests/test_conversion_result_json.py b/tests/test_conversion_result_json.py index 3f2f2e7fec..f49a26703e 100644 --- a/tests/test_conversion_result_json.py +++ b/tests/test_conversion_result_json.py @@ -16,7 +16,7 @@ from docling.document_converter import DocumentConverter, PdfFormatOption -def test_conversion_result_json_roundtrip_string(): +def test_conversion_result_json_roundtrip_string(tmp_path: Path): pdf_doc = Path("./tests/data/pdf/sources/redp5110_sampled.pdf") pipeline_options = PdfPipelineOptions() @@ -36,7 +36,7 @@ def test_conversion_result_json_roundtrip_string(): ) conv_res = doc_converter.convert(pdf_doc) - fpath: Path = Path("./test-conversion.zip") + fpath: Path = tmp_path / "test-conversion.zip" conv_res.save(filename=fpath) # returns string when no filename is given # assert isinstance(json_str, str) and len(json_str) > 0 diff --git a/tests/test_e2e_nemotron_ocr_conversion.py b/tests/test_e2e_nemotron_ocr_conversion.py index fb777dd567..1619a0965a 100644 --- a/tests/test_e2e_nemotron_ocr_conversion.py +++ b/tests/test_e2e_nemotron_ocr_conversion.py @@ -23,10 +23,11 @@ PdfPipelineOptions, ) from docling.document_converter import DocumentConverter, PdfFormatOption +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.stages.ocr.nemotron_ocr_model import ( NemotronOcrModel, - resolve_nemotronocr_language, ) +from docling.utils.ocr_language import OcrLanguageResolver from .groundtruth_paths import ( GroundTruthPaths, @@ -128,32 +129,39 @@ def get_converter(ocr_options: OcrOptions, ocr_batch_size: Optional[int] = None) @pytest.mark.parametrize( - ("req_languages", "expected"), + ("tag", "expected"), [ - # No request -> english default - (None, "english"), - ([], "english"), - # English aliases (and case / whitespace / region-tag normalization) - (["en"], "english"), - (["eng"], "english"), - (["english"], "english"), - (["EN"], "english"), - (["English"], "english"), - ([" en "], "english"), - (["en-US"], "english"), - (["en_US"], "english"), - (["en", "english", "eng"], "english"), - # Any non-english language maps to multilingual - (["de"], "multilingual"), - (["fr"], "multilingual"), - (["zh-CN"], "multilingual"), - # A single non-english language is enough to promote the whole request - (["en", "de"], "multilingual"), - (["de", "en"], "multilingual"), + # Every spelling of English canonicalizes to the same recognizer. + ("en", "english"), + ("eng", "english"), + ("EN", "english"), + (" en ", "english"), + ("en-US", "english"), + # The languages the multilingual checkpoint is trained on. + ("zh-CN", "multilingual"), + ("zh-Hant", "multilingual"), + ("ja", "multilingual"), + ("ko", "multilingual"), + ("ru", "multilingual"), + # ...and the explicit escape hatch. + ("mul", "multilingual"), ], ) -def test_nemotron_language_resolution(req_languages, expected): - assert resolve_nemotronocr_language(req_languages) == expected +def test_nemotron_language_mapping(tag, expected): + model = NemotronOcrModel.__new__(NemotronOcrModel) + assert ( + model.map_ocr_language(OcrLanguageResolver.canonicalize_ocr_language(tag)) + == expected + ) + + +@pytest.mark.parametrize("tag", ["de", "fr", "ar", "hi"]) +def test_nemotron_rejects_uncovered_language(tag): + """A language neither checkpoint serves errors instead of silently + falling back to the multilingual model, which is what used to happen.""" + model = NemotronOcrModel.__new__(NemotronOcrModel) + with pytest.raises(OcrLanguageNotSupportedError, match="mul"): + model.map_ocr_language(OcrLanguageResolver.canonicalize_ocr_language(tag)) def test_e2e_nemotron_ocr_conversions(): diff --git a/tests/test_e2e_ocr_conversion.py b/tests/test_e2e_ocr_conversion.py index 255ece1e27..ac98ce04db 100644 --- a/tests/test_e2e_ocr_conversion.py +++ b/tests/test_e2e_ocr_conversion.py @@ -77,9 +77,9 @@ def test_e2e_conversions(): (EasyOcrOptions(mode=OcrMode.LAYOUT_REGIONS), False), # Full page OCR (TesseractOcrOptions(mode=OcrMode.FULL_PAGE), True), - (TesseractOcrOptions(mode=OcrMode.FULL_PAGE, lang=["auto"]), True), + (TesseractOcrOptions(mode=OcrMode.FULL_PAGE, lang=[]), True), (TesseractCliOcrOptions(mode=OcrMode.FULL_PAGE), True), - (TesseractCliOcrOptions(mode=OcrMode.FULL_PAGE, lang=["auto"]), True), + (TesseractCliOcrOptions(mode=OcrMode.FULL_PAGE, lang=[]), True), (EasyOcrOptions(mode=OcrMode.FULL_PAGE), False), ] diff --git a/tests/test_easyocr_lang.py b/tests/test_easyocr_lang.py index a203fbdf74..8450722240 100644 --- a/tests/test_easyocr_lang.py +++ b/tests/test_easyocr_lang.py @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: The Docling Contributors # SPDX-License-Identifier: MIT -import re import zipfile from io import BytesIO from pathlib import Path @@ -10,30 +9,27 @@ from typer.testing import CliRunner from docling.cli.tools import app +from docling.datamodel.accelerator_options import AcceleratorOptions +from docling.datamodel.pipeline_options import EasyOcrOptions from docling.models.stages.ocr import easyocr_model from docling.models.stages.ocr.easyocr_model import EasyOcrModel from docling.utils.model_downloader import download_models pytestmark = pytest.mark.ml_ocr -runner = CliRunner() -_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") +# Under CI Rich thinks it has a terminal and styles the error panel, landing +# escapes between the border and the wrapped halves of a sentence. +# `TERM=dumb` turns that off, so the panel arrives as plain wrapped text. +runner = CliRunner(env={"TERM": "dumb"}) def _single_line_cli_output(output: str) -> str: - return " ".join(_ANSI_RE.sub("", output).replace("│", "").split()) - - -def test_single_line_cli_output_strips_ansi_styles() -> None: - output = "\x1b[1;33m--easyocr-lang\x1b[0m requires the 'easyocr'\n│ model" - - assert _single_line_cli_output(output) == ( - "--easyocr-lang requires the 'easyocr' model" - ) + """The error panel still wraps and draws borders: flatten it to one line.""" + return " ".join(output.replace("│", "").split()) @pytest.mark.parametrize( - ("language", "model_name"), + ("tag", "model_name"), [ ("en", "english_g2"), ("de", "latin_g2"), @@ -42,8 +38,8 @@ def test_single_line_cli_output_strips_ansi_styles() -> None: ("hi", "devanagari_g1"), ("ru", "cyrillic_g2"), ("th", "thai_g1"), - ("ch_tra", "zh_tra_g1"), - ("ch_sim", "zh_sim_g2"), + ("zh-Hant", "zh_tra_g1"), + ("zh-Hans", "zh_sim_g2"), ("ja", "japanese_g2"), ("ko", "korean_g2"), ("ta", "tamil_g1"), @@ -51,21 +47,84 @@ def test_single_line_cli_output_strips_ansi_styles() -> None: ("kn", "kannada_g2"), ], ) -def test_resolve_easyocr_language(language: str, model_name: str) -> None: - assert easyocr_model._resolve_easyocr_recognition_models([language]) == [model_name] +def test_prefetch_resolves_bcp47_to_a_checkpoint(tag: str, model_name: str) -> None: + codes = easyocr_model.resolve_easyocr_codes([tag]) + + assert easyocr_model._resolve_easyocr_recognition_models(codes) == [model_name] + + +def test_resolve_easyocr_languages_maps_to_native_codes() -> None: + """EasyOCR keeps its own vocabulary internally; only the input changed.""" + assert easyocr_model.resolve_easyocr_codes(["zh-Hant", "sr-Latn", "tg"]) == [ + "ch_tra", + "rs_latin", + "tjk", + ] + + +def test_resolve_easyocr_languages_routes_to_the_script_model() -> None: + """Each language reaches the recognition network of its own script, so the + caller names languages and never a script.""" + codes = easyocr_model.resolve_easyocr_codes(["ru", "sr-Cyrl"]) + + assert codes == ["ru", "rs_cyrillic"] + assert easyocr_model._resolve_easyocr_recognition_models(codes) == ["cyrillic_g2"] def test_resolve_easyocr_languages_deduplicates_models() -> None: - assert easyocr_model._resolve_easyocr_recognition_models( - ["de", "fr", "ch_sim", "de", "ch_sim"] - ) == ["latin_g2", "zh_sim_g2"] + codes = easyocr_model.resolve_easyocr_codes(["de", "fr", "zh-Hans", "de-AT"]) + + assert easyocr_model._resolve_easyocr_recognition_models(codes) == [ + "latin_g2", + "zh_sim_g2", + ] + + +def test_resolve_easyocr_languages_rejects_malformed_tag() -> None: + with pytest.raises(ValueError, match="BCP-47"): + easyocr_model.resolve_easyocr_codes(["xx"]) -def test_resolve_easyocr_languages_rejects_unsupported_code() -> None: +def test_resolve_easyocr_languages_rejects_uncovered_language() -> None: + """`haw` is a valid tag EasyOCR simply has no recognizer for.""" + with pytest.raises(ValueError, match="Unsupported EasyOCR language: haw"): + easyocr_model.resolve_easyocr_codes(["haw"]) + + +def test_resolve_easyocr_recognition_models_rejects_unsupported_code() -> None: with pytest.raises(ValueError, match="Unsupported EasyOCR language code: xx"): easyocr_model._resolve_easyocr_recognition_models(["xx"]) +@pytest.mark.parametrize(("lang", "expected"), [([], ["en"]), (["de"], ["de"])]) +def test_empty_lang_reaches_the_reader_as_english( + monkeypatch, lang: list[str], expected: list[str] +) -> None: + """EasyOCR has no engine default, and an empty `lang_list` is not one. + + `easyocr.Reader([])` falls back to the `latin_g2` checkpoint with only that + model's symbols as its character set, so every letter is dropped from the + recognized text -- silently. Docling names a language instead. + """ + import easyocr + + captured: list[list[str]] = [] + monkeypatch.setattr( + easyocr, + "Reader", + lambda lang_list, **kwargs: captured.append(lang_list), + ) + + EasyOcrModel( + enabled=True, + artifacts_path=None, + options=EasyOcrOptions(lang=lang), + accelerator_options=AcceleratorOptions(), + ) + + assert captured == [expected] + + def test_easyocr_downloader_supports_gen1_and_gen2_models( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -141,13 +200,13 @@ def fake_download_models(**kwargs: object) -> None: with_picture_classifier=False, with_rapidocr=False, with_easyocr=True, - easyocr_languages=["ch_sim", "ja", "ch_sim"], + easyocr_languages=["zh-Hans", "ja", "zh-CN"], ) assert len(captured_calls) == 1 assert captured_calls[0]["recognition_models"] == [ - "zh_sim_g2", "japanese_g2", + "zh_sim_g2", ] @@ -183,7 +242,7 @@ def test_model_downloader_validates_easyocr_languages_before_io( ) -> None: output_dir = tmp_path / "models" - with pytest.raises(ValueError, match="Unsupported EasyOCR language code: xx"): + with pytest.raises(ValueError, match="BCP-47"): download_models( output_dir=output_dir, with_layout=False, @@ -234,7 +293,7 @@ def fake_download_models(**kwargs: object) -> Path: "download", *model_args, "--easyocr-lang", - "ch_sim", + "zh-Hans", "--easyocr-lang", "ja", "--output-dir", @@ -245,7 +304,8 @@ def fake_download_models(**kwargs: object) -> Path: assert result.exit_code == 0, result.output assert len(captured_calls) == 1 - assert captured_calls[0]["easyocr_languages"] == ["ch_sim", "ja"] + # The CLI hands the downloader the user's tags; they are resolved there. + assert captured_calls[0]["easyocr_languages"] == ["zh-Hans", "ja"] def test_models_cli_rejects_easyocr_languages_without_easyocr( @@ -307,7 +367,5 @@ def fake_download_models(**kwargs: object) -> Path: ) assert result.exit_code == 2 - assert "Unsupported EasyOCR language code: xx" in _single_line_cli_output( - result.output - ) + assert "BCP-47" in _single_line_cli_output(result.output) assert not called diff --git a/tests/test_kserve_v2_ocr_integration.py b/tests/test_kserve_v2_ocr_integration.py index 5297b1b5f7..ad3816bfe0 100644 --- a/tests/test_kserve_v2_ocr_integration.py +++ b/tests/test_kserve_v2_ocr_integration.py @@ -34,9 +34,10 @@ KSERVE_OCR_TRANSPORTS = ["http", "grpc"] KSERVE_OCR_LANGUAGES = [ "en", - "ch", + "zh-Hans", + # PP-OCR's script recognizers, named by their own tokens. "arabic", - "korean", + "ko", "latin", ] diff --git a/tests/test_ocr_engine_language_mapping.py b/tests/test_ocr_engine_language_mapping.py new file mode 100644 index 0000000000..038b380a6c --- /dev/null +++ b/tests/test_ocr_engine_language_mapping.py @@ -0,0 +1,369 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Per-engine translation from canonical tags to native codes. + +Most of these run without any engine installed: each engine's table and mapping +are module-level or reachable on an uninitialized instance, which is what makes +the mapping reviewable at all. The last section is the exception -- what an +engine advertises depends on what is installed, so those tests need the engine. +""" + +import logging +import shutil +import sys + +import pytest + +from docling.datamodel.accelerator_options import AcceleratorOptions +from docling.datamodel.pipeline_options import ( + EasyOcrOptions, + KserveV2OcrOptions, + OcrMacOptions, + RapidOcrOptions, + TesseractCliOcrOptions, + TesseractOcrOptions, +) +from docling.exceptions import OcrLanguageNotSupportedError +from docling.models.stages.ocr.kserve_v2_ocr_model import KserveV2OcrModel +from docling.models.stages.ocr.ppocr_languages import ( + PPOCRV4_CODES, + PPOCRV5_CODES, + PPOCRV6_CODES, + ppocr_code, + ppocr_supported_tags, +) +from docling.models.stages.ocr.rapid_ocr_model import RapidOcrModel +from docling.models.stages.ocr.tesseract_utils import language_to_tesseract_code +from docling.utils.ocr_language import OcrLanguage, OcrLanguageResolver + +_ONNX_VOCABULARY = PPOCRV6_CODES | PPOCRV5_CODES | PPOCRV4_CODES +_TORCH_VOCABULARY = PPOCRV6_CODES | PPOCRV4_CODES + + +# --- PP-OCR (RapidOCR and the KServe client share one table) ---------------- + + +@pytest.mark.parametrize( + ("tag", "expected"), + [ + ("zh-Hans", "ch"), + ("zh-Hant", "chinese_cht"), + ("ja", "japan"), + ("en", "en"), + ("de", "de"), + ("sr-Latn", "rs_latin"), + # East Slavic has its own, narrower recognizer. + ("ru", "eslav"), + ("uk", "eslav"), + ("be", "eslav"), + # Any other Cyrillic language falls back to the script family. + ("sr", "cyrillic"), + ("mn", "cyrillic"), + ("el", "el"), + ("th", "th"), + ("hi", "devanagari"), + ], +) +def test_ppocr_tokens(tag: str, expected: str) -> None: + assert ( + ppocr_code(OcrLanguageResolver.canonicalize_ocr_language(tag), _ONNX_VOCABULARY) + == expected + ) + + +@pytest.mark.parametrize("token", ["latin", "cyrillic", "arabic", "devanagari"]) +def test_ppocr_script_recognizers_are_named_by_their_own_token(token: str) -> None: + """These are real PP-OCR models with no language to canonicalize to, so they + are carried through to the engine exactly as the user wrote them, once the + `native:` prefix marks them as an engine token rather than a tag.""" + language = OcrLanguageResolver.canonicalize_ocr_language(f"native:{token}") + + assert language.is_passthrough + assert ppocr_code(language, _ONNX_VOCABULARY) == token + + +def test_ppocr_kannada_georgian_collision() -> None: + """PP-OCR's `ka` is Kannada; BCP-47 `ka` is Georgian. + + Kannada must reach the `ka` recognizer, and Georgian must *not* -- it has no + PP-OCR model at all, and silently serving it the Kannada one is the bug this + guards. + """ + assert ( + ppocr_code( + OcrLanguageResolver.canonicalize_ocr_language("kn"), _TORCH_VOCABULARY + ) + == "ka" + ) + assert ( + ppocr_code( + OcrLanguageResolver.canonicalize_ocr_language("ka"), _TORCH_VOCABULARY + ) + is None + ) + assert ( + ppocr_code( + OcrLanguageResolver.canonicalize_ocr_language("ka"), _ONNX_VOCABULARY + ) + is None + ) + + +def test_ppocr_has_no_multilingual_model() -> None: + assert ( + ppocr_code( + OcrLanguageResolver.canonicalize_ocr_language("mul"), _ONNX_VOCABULARY + ) + is None + ) + + +def test_ppocr_non_default_script_uses_the_family() -> None: + """PP-OCR's `az` and `uz` are the Latin ones, so a Cyrillic request for the + same language must not silently pick the Latin recognizer.""" + assert ( + ppocr_code( + OcrLanguageResolver.canonicalize_ocr_language("az"), _ONNX_VOCABULARY + ) + == "az" + ) + assert ( + ppocr_code( + OcrLanguageResolver.canonicalize_ocr_language("az-Cyrl"), _ONNX_VOCABULARY + ) + == "cyrillic" + ) + assert ( + ppocr_code( + OcrLanguageResolver.canonicalize_ocr_language("uz-Cyrl"), _ONNX_VOCABULARY + ) + == "cyrillic" + ) + + +def test_ppocr_supported_tags_are_canonical() -> None: + tags = ppocr_supported_tags(_ONNX_VOCABULARY) + + assert "zh-Hans" in tags + # Languages are rendered back as tags, never as PP-OCR tokens... + assert "ch" not in tags + # ...but a script recognizer is named back as itself: that is what selects it. + assert "cyrillic" in tags + + +# --- RapidOCR backend routing ---------------------------------------------- + + +def _rapid_model(backend: str, lang: list[str]) -> RapidOcrModel: + model = RapidOcrModel.__new__(RapidOcrModel) + model.options = RapidOcrOptions(backend=backend, lang=lang) + model.languages = tuple( + OcrLanguageResolver.canonicalize_ocr_language(tag) for tag in model.options.lang + ) + return model + + +@pytest.mark.parametrize("backend", ["onnxruntime", "torch"]) +def test_rapidocr_georgian_is_a_coverage_error_on_every_backend(backend: str) -> None: + """Georgian has no PP-OCR recognizer on any backend. + + It has to be asked for as `ka-Geor`: a bare `ka` given to RapidOCR is PP-OCR's + own token for Kannada, which is the reading RapidOCR users expect. + """ + model = _rapid_model(backend, ["ka-Geor"]) + + with pytest.raises(OcrLanguageNotSupportedError) as excinfo: + model.resolve_ocr_languages() + + message = str(excinfo.value) + assert "ka-Geor" in message + assert backend in message + # The message must name what the user *can* ask for. + assert "Supported:" in message + + +def test_rapidocr_native_ka_is_ppocr_kannada() -> None: + """`native:ka` names PP-OCR's Kannada recognizer; bare `ka` is BCP-47 Georgian.""" + options = RapidOcrOptions(backend="torch", lang=["native:ka"]) + assert options.lang == ["native:ka"] + assert _rapid_model("torch", ["native:ka"]).resolve_ocr_languages() == ["ka"] + + +def test_rapidocr_warns_and_truncates_extra_languages( + caplog: pytest.LogCaptureFixture, +) -> None: + model = _rapid_model("onnxruntime", ["de", "fr", "en"]) + + with caplog.at_level(logging.WARNING): + assert model.resolve_ocr_languages() == ["de"] + + warning = caplog.text + assert "de-Latn" in warning + assert "fr-Latn" in warning and "en-Latn" in warning + assert "preference" in warning + + +# --- KServe v2 -------------------------------------------------------------- + +# KServe canonicalizes nothing: the deployed model is the only authority on the +# languages it serves, so `lang` is neither validated nor mapped, only truncated +# to the one value the request carries. + + +def test_kserve_sends_the_engines_own_code_untouched() -> None: + """`chi_sim` is another engine's code and `auto` is retired, yet both survive: + only the deployment knows what it serves.""" + options = KserveV2OcrOptions(url="http://localhost:8000", lang=["chi_sim", "auto"]) + + assert options.lang == ["chi_sim", "auto"] + + +def test_kserve_default_lang_is_not_canonicalized() -> None: + options = KserveV2OcrOptions(url="http://localhost:8000") + + assert options.lang == ["english", "chinese"] + + +def test_kserve_warns_and_sends_the_first_language( + caplog: pytest.LogCaptureFixture, +) -> None: + """One language fits the request; the rest are dropped, but never silently.""" + options = KserveV2OcrOptions( + url="http://localhost:8000", transport="http", lang=["japan", "korean"] + ) + model = KserveV2OcrModel.__new__(KserveV2OcrModel) + + with caplog.at_level(logging.WARNING): + KserveV2OcrModel.__init__( + model, + enabled=True, + artifacts_path=None, + options=options, + accelerator_options=AcceleratorOptions(), + ) + + assert model._lang == "japan" + assert "japan" in caplog.text and "korean" in caplog.text + + +def test_the_opt_out_does_not_leak_to_other_engines() -> None: + """Only KServe skips canonicalization; a sibling still rewrites its tags.""" + assert RapidOcrOptions(lang=["deu"]).lang == ["de-Latn"] + + +# --- Tesseract -------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("tag", "expected"), + [ + # The vocabulary *is* ISO 639-2/T, so most tags need no table entry. + ("de", "deu"), + ("fr", "fra"), + ("el", "ell"), + ("cs", "ces"), + ("en", "eng"), + ("kn", "kan"), + # Georgian is `kat` here -- no collision, unlike PP-OCR. + ("ka", "kat"), + # ...and the deviations that do. + ("zh-Hans", "chi_sim"), + ("zh-Hant", "chi_tra"), + ("sr", "srp"), + ("sr-Latn", "srp_latn"), + ("az-Cyrl", "aze_cyrl"), + ("az", "aze"), + ("uz-Cyrl", "uzb_cyrl"), + ("ku", "kmr"), + ("nb", "nor"), + ("nn", "nor"), + ], +) +def test_tesseract_language_names(tag: str, expected: str) -> None: + assert ( + language_to_tesseract_code(OcrLanguageResolver.canonicalize_ocr_language(tag)) + == expected + ) + + +def test_tesseract_script_files_pass_through_verbatim() -> None: + """A script file names its traineddata directly, in the install's spelling.""" + assert ( + language_to_tesseract_code(OcrLanguage(native="script/Latin")) == "script/Latin" + ) + assert language_to_tesseract_code(OcrLanguage(native="Cyrillic")) == "Cyrillic" + + +def test_tesseract_has_no_file_for_mul() -> None: + """`mul` has no tessdata equivalent; an empty list drives per-page OSD.""" + assert ( + language_to_tesseract_code(OcrLanguageResolver.canonicalize_ocr_language("mul")) + is None + ) + + +# --- what an engine advertises must be requestable -------------------------- +# +# `supported_ocr_languages()` fills the "Supported:" line of +# `OcrLanguageNotSupportedError`, so it is a list users copy from. Every tag in +# it therefore has to survive being asked for again -- which is exactly what +# three engines got wrong: EasyOCR offered `av-Cyrl`/`ce-Cyrl` under codes it +# does not have, Tesseract offered the `script/*_vert` files the resolver +# refuses, and ocrmac offered Vision's own `vi-VT`, which is not a tag at all. + + +def _assert_every_advertised_tag_is_requestable(model) -> None: + advertised = model.supported_ocr_languages() + assert advertised, "the engine reported no languages at all" + unusable = [] + for tag in advertised: + try: + model.map_ocr_language(OcrLanguageResolver.canonicalize_ocr_language(tag)) + except (ValueError, OcrLanguageNotSupportedError) as exc: + unusable.append((tag, str(exc))) + assert not unusable + + +def test_easyocr_advertises_only_languages_it_serves() -> None: + pytest.importorskip("easyocr") + from docling.models.stages.ocr.easyocr_model import EasyOcrModel + + model = EasyOcrModel( + enabled=False, + artifacts_path=None, + options=EasyOcrOptions(), + accelerator_options=AcceleratorOptions(), + ) + + _assert_every_advertised_tag_is_requestable(model) + + +def test_tesseract_advertises_only_languages_it_serves() -> None: + if shutil.which("tesseract") is None: + pytest.skip("tesseract binary not installed") + from docling.models.stages.ocr.tesseract_ocr_cli_model import TesseractOcrCliModel + + model = TesseractOcrCliModel( + enabled=True, + artifacts_path=None, + options=TesseractCliOcrOptions(lang=["en"]), + accelerator_options=AcceleratorOptions(), + ) + + _assert_every_advertised_tag_is_requestable(model) + + +@pytest.mark.skipif(sys.platform != "darwin", reason="ocrmac is macOS-only") +def test_ocrmac_advertises_only_languages_it_serves() -> None: + pytest.importorskip("ocrmac") + from docling.models.stages.ocr.ocr_mac_model import OcrMacModel + + model = OcrMacModel( + enabled=True, + artifacts_path=None, + options=OcrMacOptions(), + accelerator_options=AcceleratorOptions(), + ) + + _assert_every_advertised_tag_is_requestable(model) diff --git a/tests/test_ocr_language.py b/tests/test_ocr_language.py new file mode 100644 index 0000000000..ccfb346672 --- /dev/null +++ b/tests/test_ocr_language.py @@ -0,0 +1,583 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Docling's OCR language policy: canonicalize to BCP-47, or refuse. + +These assert docling decisions -- drop the region, keep the script, never +reject `und`, reject the retired engine vocabularies -- rather than langcodes +behaviour. + +Four sections, following one user-supplied string all the way to a canonical tag: +the resolver itself, the engine-native vocabularies it accepts alongside BCP-47, +the `OcrOptions` validator that calls it, and the `--ocr-lang` CLI flag that +feeds that validator. +""" + +import warnings +from pathlib import Path +from typing import Any + +import pytest +from pydantic import ValidationError +from typer.testing import CliRunner + +from docling.cli.main import app +from docling.datamodel.accelerator_options import AcceleratorOptions +from docling.datamodel.base_models import InputFormat +from docling.datamodel.pipeline_options import ( + EasyOcrOptions, + NemotronOcrOptions, + OcrAutoOptions, + OcrMacOptions, + OcrMode, + RapidOcrOptions, + TesseractCliOcrOptions, + TesseractOcrOptions, +) +from docling.datamodel.settings import DEFAULT_PAGE_RANGE +from docling.models.stages.ocr.auto_ocr_model import OcrAutoModel +from docling.models.stages.ocr.easyocr_model import EasyOcrModel +from docling.models.stages.ocr.tesseract_utils import language_to_tesseract_code +from docling.utils.ocr_language import ( + OcrLanguage, + OcrLanguageResolver, +) + + +def _canonical_tags(values: list[str]) -> list[str]: + """The tags `OcrOptions.lang` would store for `values`.""" + return [ + language.tag + for language in OcrLanguageResolver.canonicalize_ocr_languages(values) + ] + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + # ISO 639-2/B and /T both fold onto the 639-1 subtag; region is dropped. + ("de", "de-Latn"), + ("de-DE", "de-Latn"), + ("deu", "de-Latn"), + ("ger", "de-Latn"), + ("DE", "de-Latn"), + ("en", "en-Latn"), + ("en-US", "en-Latn"), + ("eng", "en-Latn"), + # Simplified vs Traditional is a script distinction no ISO 639 code has. + ("zh", "zh-Hans"), + ("zh-CN", "zh-Hans"), + ("zh-Hans", "zh-Hans"), + ("zho", "zh-Hans"), + ("zh-TW", "zh-Hant"), + ("zh-HK", "zh-Hant"), + ("zh-Hant", "zh-Hant"), + # Likely-subtags supply the script the user left out. + ("sr", "sr-Cyrl"), + ("sr-Latn", "sr-Latn"), + ("pa", "pa-Guru"), + ("pa-IN", "pa-Guru"), + ("pa-PK", "pa-Arab"), + ("ja", "ja-Jpan"), + ("jpn", "ja-Jpan"), + ("ru", "ru-Cyrl"), + # The reserved tag passes through untouched. + ("mul", "mul"), + (" de ", "de-Latn"), + ], +) +def test_canonicalization(value: str, expected: str) -> None: + assert OcrLanguageResolver.canonicalize_ocr_language(value).tag == expected + + +@pytest.mark.parametrize("value", ["und", "und-Latn", "und-latn", "und-Cyrl"]) +def test_undetermined_tags_are_rejected(value: str) -> None: + """Docling has no "undetermined" language and no script families. + + `Language.get("und-Latn").maximize()` is `en-Latn-US`, so accepting these + would silently turn "any Latin-script document" into "English". The empty + list carries the "let the engine decide" meaning instead. + """ + with pytest.raises(ValueError, match="no script families"): + OcrLanguageResolver.canonicalize_ocr_language(value) + + +@pytest.mark.parametrize( + ("value", "hint"), + [ + ("chinese", "zh-Hans"), + # `ch` is valid BCP-47 for Chamorro, so validity alone would accept it. + ("ch", "zh-Hans"), + ("ch_sim", "zh-Hans"), + ("chi_sim", "zh-Hans"), + ("chi_tra", "zh-Hant"), + ("chinese_cht", "zh-Hant"), + ("english", "en"), + ("japan", "ja"), + ("korean", "ko"), + ("multilingual", "mul"), + ("eslav", "ru"), + ("rs_latin", "sr-Latn"), + ("tjk", "tg"), + ("aze_cyrl", "az-Cyrl"), + ], +) +def test_retired_engine_tokens_name_their_replacement(value: str, hint: str) -> None: + with pytest.raises(ValueError) as excinfo: + OcrLanguageResolver.canonicalize_ocr_language(value) + + assert f"'{hint}'" in str(excinfo.value) + + +@pytest.mark.parametrize("value", ["klingon", "", " ", "de-DE-DE", "zz"]) +def test_malformed_tags_raise(value: str) -> None: + with pytest.raises(ValueError): + OcrLanguageResolver.canonicalize_ocr_language(value) + + +def test_chamorro_is_not_chinese() -> None: + """RapidOCR's `ch` means Chinese, but BCP-47 `ch` is Chamorro. + + Accepting it would resolve to the wrong language and surface as a confusing + coverage error much later, so it is rejected up front. + """ + with pytest.raises(ValueError, match="zh-Hans"): + OcrLanguageResolver.canonicalize_ocr_language("ch") + + +def test_reserved_tag_must_stand_alone() -> None: + for combination in (["mul", "en"], ["en", "mul"]): + with pytest.raises(ValueError, match="on its own"): + OcrLanguageResolver.canonicalize_ocr_languages(combination) + + +@pytest.mark.parametrize("value", ["zxx", "ZXX", " zxx "]) +def test_no_linguistic_content_is_not_an_ocr_language(value: str) -> None: + """`zxx` used to disable the engine; skipping OCR is a pipeline switch, not a + language, and langcodes would otherwise read the tag as `zxx-Latn`.""" + with pytest.raises(ValueError, match="do_ocr=False"): + OcrLanguageResolver.canonicalize_ocr_language(value) + + +def test_empty_list_means_the_engine_decides() -> None: + """No tag carries that meaning any more, so the empty list has to.""" + assert OcrLanguageResolver.canonicalize_ocr_languages([]) == [] + + +@pytest.mark.parametrize( + "value", ["auto", "osd", "latin", "cyrillic", "arabic", "devanagari", "bengali"] +) +def test_engine_decides_and_script_tokens_point_at_the_empty_list(value: str) -> None: + """These named a script or an auto mode, neither of which is a language. + + PP-OCR defines four of them as real recognizers, so they resolve as + passthroughs for that engine and only reach this path for everyone else. + """ + with pytest.raises(ValueError, match="leave the OCR language list empty"): + OcrLanguageResolver.canonicalize_ocr_language(value) + + +def test_duplicates_collapse_and_order_is_preserved() -> None: + """Order is preference order for engines that join languages (Tesseract `+`).""" + assert _canonical_tags(["fr", "de", "fr-FR", "en-GB", "deu"]) == [ + "fr-Latn", + "de-Latn", + "en-Latn", + ] + + +def test_canonicalization_is_idempotent() -> None: + once = _canonical_tags(["deu", "zh-TW", "sr-Latn", "pa-PK"]) + + assert _canonical_tags(once) == once + + +def test_has_default_script_separates_the_script_variants() -> None: + """Engines key on this to decide whether the primary subtag is enough.""" + assert OcrLanguageResolver.canonicalize_ocr_language("de").has_default_script + assert OcrLanguageResolver.canonicalize_ocr_language("sr").has_default_script + assert not OcrLanguageResolver.canonicalize_ocr_language( + "sr-Latn" + ).has_default_script + assert not OcrLanguageResolver.canonicalize_ocr_language( + "az-Cyrl" + ).has_default_script + + +def test_ocr_language_is_hashable() -> None: + """Engines key dicts and caches on the canonical pair.""" + assert {OcrLanguage(bcp47_language="de", bcp47_script="Latn")} == { + OcrLanguageResolver.canonicalize_ocr_language("de-DE") + } + + +def test_match_against_a_region_bearing_vocabulary() -> None: + """Apple Vision's own vocabulary is BCP-47 with regions.""" + supported = ["en-US", "fr-FR", "de-DE", "pt-BR", "zh-Hans"] + + assert ( + OcrLanguageResolver.match_ocr_language( + OcrLanguageResolver.canonicalize_ocr_language("de"), supported + ) + == "de-DE" + ) + assert ( + OcrLanguageResolver.match_ocr_language( + OcrLanguageResolver.canonicalize_ocr_language("pt"), supported + ) + == "pt-BR" + ) + assert ( + OcrLanguageResolver.match_ocr_language( + OcrLanguageResolver.canonicalize_ocr_language("zh-CN"), supported + ) + == "zh-Hans" + ) + assert ( + OcrLanguageResolver.match_ocr_language( + OcrLanguageResolver.canonicalize_ocr_language("th"), supported + ) + is None + ) + + +# --- engine-native vocabularies -------------------------------------------- + + +@pytest.mark.parametrize( + "token", ["jpn_vert", "chi_tra_vert", "script/HanS_vert", "ita_old", "equ"] +) +def test_tokens_no_language_tag_can_express_are_refused(token: str) -> None: + """Rather than silently resolving to a different recognizer. + + `ita_old` in particular used to canonicalize to plain `it-Latn`, quietly + selecting the modern Italian model for a historical-orthography request. + """ + with pytest.raises(ValueError, match="cannot address by language tag"): + OcrLanguageResolver.canonicalize_ocr_language(token) + + +@pytest.mark.parametrize( + "value", + [ + "klingon", # not a valid BCP-47 tag + "chi_sim", # a legacy hint + "osd", # an auto token + "script/HanS_vert", # vertical text + ], +) +def test_canonicalize_can_answer_none_instead_of_raising(value: str) -> None: + """The four routes into the failure guard, for the vocabulary builders. + + They advertise what an engine serves, where a rejection reason never reaches + a user; the default keeps the `ValueError` that carries one. + """ + assert ( + OcrLanguageResolver.canonicalize_ocr_language(value, raise_exception=False) + is None + ) + with pytest.raises(ValueError): + OcrLanguageResolver.canonicalize_ocr_language(value) + + +def test_unrepresentable_tokens_are_scoped_to_the_engine_that_owns_them() -> None: + """`equ` asked of RapidOCR is better served by "not a valid tag".""" + with pytest.raises(ValueError, match="BCP-47"): + OcrLanguageResolver.canonicalize_ocr_language("equ") + + +@pytest.mark.parametrize( + ("options_cls", "native", "expected"), + [ + (RapidOcrOptions, ["native:ch"], ["native:ch"]), + (RapidOcrOptions, ["native:chinese_cht"], ["native:chinese_cht"]), + (TesseractOcrOptions, ["native:chi_tra"], ["native:chi_tra"]), + # Only the prefix is case insensitive: a tessdata script file is + # TitleCase and has to reach the engine spelled as it is installed. + ( + TesseractCliOcrOptions, + ["native:script/Cyrillic"], + ["native:script/Cyrillic"], + ), + ( + EasyOcrOptions, + ["native:ch_sim", "native:ang"], + ["native:ch_sim", "native:ang"], + ), + (NemotronOcrOptions, ["native:multilingual"], ["native:multilingual"]), + (OcrAutoOptions, ["native:chinese"], ["native:chinese"]), + # ocrmac has no native vocabulary: Vision's codes already are BCP-47. + (OcrMacOptions, ["en-US"], ["en-Latn"]), + ], +) +def test_options_accept_their_own_engines_codes( + options_cls: type, native: list[str], expected: list[str] +) -> None: + """A `native:` token is stored as written, so revalidating `lang` cannot move it.""" + assert options_cls(lang=native).lang == expected + + +def test_options_reject_a_clashing_token_when_no_engine_is_chosen() -> None: + with pytest.raises(ValidationError, match="zh-Hans"): + OcrAutoOptions(lang=["ch"]) + + +def test_tesseract_fraktur_keeps_its_own_traineddata() -> None: + """`de-Latf` used to flatten to `deu` through to_alpha3(), losing Fraktur.""" + assert ( + language_to_tesseract_code( + OcrLanguageResolver.canonicalize_ocr_language("de-Latf") + ) + == "deu_latf" + ) + assert ( + language_to_tesseract_code(OcrLanguageResolver.canonicalize_ocr_language("frk")) + == "deu_latf" + ) + + +# --- the options validator -------------------------------------------------- +# +# `OcrOptions` declares one `@field_validator("lang")`, but every engine subclass +# *redefines* `lang` with its own default and its own `ConfigDict`. The design +# assumes pydantic collects validators by field name across the MRO and merges +# `model_config` down it. Both are asserted here, because a silent regression +# would let an engine-native token through unvalidated. + +_OPTION_CLASSES = [ + OcrAutoOptions, + RapidOcrOptions, + NemotronOcrOptions, + EasyOcrOptions, + TesseractCliOcrOptions, + TesseractOcrOptions, + OcrMacOptions, +] + + +def _build(cls, **kwargs): + return cls(**kwargs) + + +@pytest.mark.parametrize("cls", _OPTION_CLASSES) +def test_base_validator_fires_on_every_subclass(cls) -> None: + options = _build(cls, lang=["deu", "en-US", "zh-TW"]) + + assert options.lang == ["de-Latn", "en-Latn", "zh-Hant"] + + +@pytest.mark.parametrize("cls", _OPTION_CLASSES) +def test_defaults_are_already_canonical(cls) -> None: + """`validate_default=True` makes this an assertion, not a rewrite.""" + default = _build(cls).lang + + assert _canonical_tags(default) == default + + +@pytest.mark.parametrize("cls", _OPTION_CLASSES) +def test_retired_tokens_are_rejected(cls) -> None: + with pytest.raises(ValidationError, match="no script families"): + _build(cls, lang=["auto"]) + + +@pytest.mark.parametrize("cls", _OPTION_CLASSES) +def test_empty_lang_is_accepted(cls) -> None: + """An empty list is how "let the engine decide" is spelled.""" + assert _build(cls, lang=[]).lang == [] + + +def test_assignment_is_validated() -> None: + """The SDK mutation path documented in the FAQ goes through the validator. + + Proves `validate_assignment` on the base survives the subclass `ConfigDict`. + """ + options = EasyOcrOptions() + + options.lang = ["fra", "de-DE"] + assert options.lang == ["fr-Latn", "de-Latn"] + + with pytest.raises(ValidationError, match="zh-Hans"): + options.lang = ["chinese"] + + +def test_force_full_page_ocr_bridge_survives_validate_assignment() -> None: + """The deprecated flag assigns `mode` from inside a model validator, which + `validate_assignment` re-enters; it must settle rather than recurse.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + options = EasyOcrOptions(force_full_page_ocr=True) + assert options.mode is OcrMode.FULL_PAGE + + options.lang = ["en"] + assert options.mode is OcrMode.FULL_PAGE + + options.scale = 2.0 + assert options.mode is OcrMode.FULL_PAGE + + +def test_serialized_options_round_trip() -> None: + options = TesseractCliOcrOptions(lang=["fra", "deu"]) + + restored = TesseractCliOcrOptions.model_validate(options.model_dump()) + + assert restored.lang == options.lang == ["fr-Latn", "de-Latn"] + + +# --- the CLI ---------------------------------------------------------------- +# +# `--ocr-lang`: construction, canonicalization, and error reporting. + +# TERM=dumb disables the Rich styling in the CI +runner = CliRunner(env={"TERM": "dumb"}) + +_SOURCE = "./tests/data/pdf/sources/2305.03393v1-pg9.pdf" + + +def _flat_cli_output(output: str) -> str: + """The error box still wraps and draws borders: flatten it to one line.""" + return " ".join(output.replace("│", "").split()) + + +def _capture_ocr_options(monkeypatch, extra_args: list[str], tmp_path: Path): + captured: dict[str, Any] = {} + + class _FakeDocumentConverter: + def __init__(self, *, allowed_formats, format_options): + pdf_option = format_options[InputFormat.PDF] + captured["ocr_options"] = pdf_option.pipeline_options.ocr_options + + def convert_all( + self, + input_doc_paths, + headers=None, + raises_on_error=False, + page_range=DEFAULT_PAGE_RANGE, + ): + return [] + + monkeypatch.setattr( + "docling.document_converter.DocumentConverter", _FakeDocumentConverter + ) + result = runner.invoke( + app, [_SOURCE, "--output", str(tmp_path / "out"), *extra_args] + ) + return result, captured.get("ocr_options") + + +def test_ocr_lang_reaches_the_options(monkeypatch, tmp_path: Path) -> None: + """The CLI constructs the options with `lang=`; it used to assign afterwards, + which bypassed validation entirely.""" + result, ocr_options = _capture_ocr_options( + monkeypatch, ["--ocr-engine", "easyocr", "--ocr-lang", "zh-Hant"], tmp_path + ) + + assert result.exit_code == 0, result.output + assert ocr_options.lang == ["zh-Hant"] + + +def test_ocr_lang_strips_whitespace(monkeypatch, tmp_path: Path) -> None: + result, ocr_options = _capture_ocr_options( + monkeypatch, ["--ocr-engine", "easyocr", "--ocr-lang", "en, de"], tmp_path + ) + + assert result.exit_code == 0, result.output + assert ocr_options.lang == ["en-Latn", "de-Latn"] + + +def test_an_empty_ocr_lang_asks_the_engine_to_choose( + monkeypatch, tmp_path: Path +) -> None: + """`--ocr-lang ""` is the only way the CLI can say `lang=[]`, which is what + reaches Tesseract's per-page script detection -- the mode the retired + `--ocr-lang auto` used to select. Omitting the option is a different + request: the engine's own default languages.""" + result, ocr_options = _capture_ocr_options( + monkeypatch, ["--ocr-engine", "easyocr", "--ocr-lang", ""], tmp_path + ) + + assert result.exit_code == 0, result.output + assert ocr_options.lang == [] + + _, defaulted = _capture_ocr_options( + monkeypatch, ["--ocr-engine", "easyocr"], tmp_path + ) + + assert defaulted.lang == EasyOcrOptions().lang + + +def test_ocr_lang_defaults_to_the_engine_default(monkeypatch, tmp_path: Path) -> None: + result, ocr_options = _capture_ocr_options( + monkeypatch, ["--ocr-engine", "rapidocr"], tmp_path + ) + + assert result.exit_code == 0, result.output + assert ocr_options.lang == ["zh-Hans"] + + +@pytest.mark.parametrize( + ("value", "hint"), + [ + # `ch` is RapidOCR's token for Chinese, but the default engine is `auto`, + # which has no engine to prefer that reading over BCP-47's Chamorro. + ("ch", "zh-Hans"), + ("auto", "leave the OCR language list empty"), + ("klingon", "BCP-47"), + ], +) +def test_retired_or_malformed_ocr_lang_fails_with_a_hint( + tmp_path: Path, value: str, hint: str +) -> None: + result = runner.invoke( + app, [_SOURCE, "--output", str(tmp_path / "out"), "--ocr-lang", value] + ) + + assert result.exit_code != 0 + assert hint in _flat_cli_output(result.output) + + +@pytest.mark.parametrize( + ("engine", "value", "expected"), + [ + ("rapidocr", "native:ch", ["native:ch"]), + ("rapidocr", "native:chinese_cht", ["native:chinese_cht"]), + ("tesseract", "native:chi_tra", ["native:chi_tra"]), + ("easyocr", "native:ch_sim", ["native:ch_sim"]), + # No engine named on the command line. + (None, "native:chinese", ["native:chinese"]), + ], +) +def test_engine_native_ocr_lang_is_accepted( + monkeypatch, + tmp_path: Path, + engine: str | None, + value: str, + expected: list[str], +) -> None: + """The selected engine's own codes work, stored exactly as they were written.""" + args = ["--ocr-lang", value] + if engine is not None: + args = ["--ocr-engine", engine, *args] + + result, ocr_options = _capture_ocr_options(monkeypatch, args, tmp_path) + + assert result.exit_code == 0, result.output + assert ocr_options.lang == expected + + +def test_another_engines_native_code_is_rejected(tmp_path: Path) -> None: + """`chi_sim` is Tesseract's; asking RapidOCR for it is a mistake worth naming.""" + result = runner.invoke( + app, + [ + _SOURCE, + "--output", + str(tmp_path / "out"), + "--ocr-engine", + "rapidocr", + "--ocr-lang", + "chi_sim", + ], + ) + + assert result.exit_code != 0 + assert "zh-Hans" in _flat_cli_output(result.output) diff --git a/tests/test_ocr_language_behavior.py b/tests/test_ocr_language_behavior.py new file mode 100644 index 0000000000..b628415e53 --- /dev/null +++ b/tests/test_ocr_language_behavior.py @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Auto-engine selection, driven by the requested OCR language. + +`OcrAutoOptions` is the one place where a language tag changes *which engine +runs*, not merely which recognizer it loads, and deciding that means probing the +installed engines for real. The parsing rules that settle before any engine +loads are in `test_ocr_language.py`. +""" + +import logging +import sys + +import pytest + +from docling.datamodel.accelerator_options import AcceleratorOptions +from docling.datamodel.pipeline_options import OcrAutoOptions +from docling.exceptions import OcrLanguageNotSupportedError +from docling.models.stages.ocr.auto_ocr_model import OcrAutoModel + +pytestmark = pytest.mark.ml_ocr + +# Amharic: a valid tag written in a script none of docling's engines recognize. +_UNSERVABLE_TAG = "am" + + +def _auto_model(lang: list[str]) -> OcrAutoModel: + return OcrAutoModel( + enabled=True, + artifacts_path=None, + options=OcrAutoOptions(lang=lang), + accelerator_options=AcceleratorOptions(), + ) + + +def test_auto_gives_the_delegate_the_users_language() -> None: + model = _auto_model(["zh-Hant"]) + + assert model._engine is not None + assert model._engine.options.lang == ["zh-Hant"] + + +@pytest.mark.skipif(sys.platform != "darwin", reason="ocrmac is macOS-only") +def test_auto_falls_through_an_engine_that_cannot_serve_the_language( + caplog: pytest.LogCaptureFixture, +) -> None: + """Apple Vision ships no Devanagari recognizer, so auto must move on rather + than fail -- picking an *available* engine is the whole contract of `auto`.""" + pytest.importorskip("ocrmac") + + with caplog.at_level(logging.INFO): + model = _auto_model(["hi"]) + + assert "skipping ocrmac" in caplog.text + assert model._engine is not None + assert not isinstance(model._engine, type(model)) + + +def test_auto_reports_every_candidate_when_none_can_serve_the_language() -> None: + """The aggregated error replaces a bare "No OCR engine found." warning.""" + with pytest.raises(OcrLanguageNotSupportedError) as excinfo: + _auto_model([_UNSERVABLE_TAG]) + + message = str(excinfo.value) + assert "am-Ethi" in message + # Every candidate is named with the reason it was passed over. + assert "No installed engine can serve it" in message diff --git a/tests/test_ocr_utils.py b/tests/test_orientation.py similarity index 100% rename from tests/test_ocr_utils.py rename to tests/test_orientation.py diff --git a/tests/test_rapid_ocr_lang.py b/tests/test_rapid_ocr_lang.py index e6e4a1a624..0086839127 100644 --- a/tests/test_rapid_ocr_lang.py +++ b/tests/test_rapid_ocr_lang.py @@ -5,13 +5,17 @@ from pathlib import Path import pytest +from pydantic import ValidationError from docling.datamodel.accelerator_options import AcceleratorOptions from docling.datamodel.pipeline_options import RapidOcrOptions from docling.datamodel.settings import settings +from docling.exceptions import OcrLanguageNotSupportedError +from docling.models.stages.ocr.ppocr_languages import ppocr_supported_tags from docling.models.stages.ocr.rapid_ocr_model import ( RapidOcrModel, _parse_rapidocr_model_spec, + _rapidocr_vocabulary, _resolve_rapidocr, ) from docling.utils.model_downloader import download_models @@ -83,9 +87,9 @@ def _build( def _resolved(lang: str, backend: str): - """The (version, registry token) pair the assertions below care about.""" + """The (version, registry code) pair the assertions below care about.""" spec = _resolve_rapidocr(lang, backend) - return spec.ppocr_version, spec.rapidocr_lang_token + return spec.ppocr_version, spec.rapidocr_code def test_resolve_populates_the_whole_spec() -> None: @@ -93,23 +97,23 @@ def test_resolve_populates_the_whole_spec() -> None: spec = _resolve_rapidocr("zh", "onnxruntime") assert spec.backend == "onnxruntime" - # The user's token is preserved verbatim, the registry token is normalized. + # The user's spelling is preserved verbatim, the registry code is normalized. assert spec.user_lang == "zh" - assert spec.rapidocr_lang_token == "ch" + assert spec.rapidocr_code == "ch" assert spec.ppocr_version == OCRVersion.PPOCRV6 def test_resolve_defaults_to_ppocrv6_chinese() -> None: from rapidocr.utils.typings import OCRVersion - assert _resolved("chinese", "onnxruntime") == (OCRVersion.PPOCRV6, "ch") + assert _resolved("zh-Hans", "onnxruntime") == (OCRVersion.PPOCRV6, "ch") assert _resolved("zh", "onnxruntime") == (OCRVersion.PPOCRV6, "ch") def test_resolve_english_and_latin_use_ppocrv6() -> None: from rapidocr.utils.typings import OCRVersion - assert _resolved("english", "onnxruntime") == (OCRVersion.PPOCRV6, "en") + assert _resolved("en", "onnxruntime") == (OCRVersion.PPOCRV6, "en") assert _resolved("en", "torch") == (OCRVersion.PPOCRV6, "en") assert _resolved("de", "onnxruntime") == (OCRVersion.PPOCRV6, "de") assert _resolved("fr", "onnxruntime") == (OCRVersion.PPOCRV6, "fr") @@ -120,17 +124,40 @@ def test_resolve_script_families_route_by_backend() -> None: # onnxruntime/openvino/paddle -> PP-OCRv5 assert _resolved("th", "onnxruntime") == (OCRVersion.PPOCRV5, "th") - assert _resolved("cyrillic", "onnxruntime") == (OCRVersion.PPOCRV5, "cyrillic") + assert _resolved("native:cyrillic", "onnxruntime") == ( + OCRVersion.PPOCRV5, + "cyrillic", + ) # torch -> PP-OCRv4 - assert _resolved("arabic", "torch") == (OCRVersion.PPOCRV4, "arabic") + assert _resolved("native:arabic", "torch") == (OCRVersion.PPOCRV4, "arabic") + # Devanagari picks the backbone its backend can reach. + assert _resolved("hi", "onnxruntime") == (OCRVersion.PPOCRV5, "devanagari") + assert _resolved("hi", "torch") == (OCRVersion.PPOCRV4, "devanagari") -def test_resolve_raises_on_unsupported_language() -> None: - with pytest.raises(ValueError): +def test_resolve_rejects_a_malformed_tag() -> None: + with pytest.raises(ValueError, match="BCP-47"): _resolve_rapidocr("klingon", "onnxruntime") + + +def test_resolve_raises_on_unsupported_language() -> None: # Thai is a PP-OCRv5 language, not served by the torch PP-OCRv4 backbone. - with pytest.raises(ValueError): + with pytest.raises(OcrLanguageNotSupportedError): _resolve_rapidocr("th", "torch") + # PP-OCR has no Georgian recognizer; its `ka` is Kannada. + with pytest.raises(OcrLanguageNotSupportedError): + _resolve_rapidocr("ka-Geor", "onnxruntime") + + +@pytest.mark.parametrize("backend", ["onnxruntime", "openvino", "paddle", "torch"]) +def test_resolve_kannada_falls_back_to_ppocrv4_on_every_backend(backend: str) -> None: + from rapidocr.utils.typings import OCRVersion + + # PP-OCR serves Kannada only on the v4 backbone, so every backend has to + # reach past its own v5/v6 set for it -- `ka` is the one code v5 lacks. + assert _resolved("kn", backend) == (OCRVersion.PPOCRV4, "ka") + # ...and it is advertised, so the coverage error never names it. + assert "kn-Knda" in ppocr_supported_tags(_rapidocr_vocabulary(backend)) # --- model selection / pinned paths ----------------------------------------- @@ -156,9 +183,9 @@ def test_rapidocr_default_onnx_uses_ppocrv6(monkeypatch, tmp_path: Path) -> None def test_rapidocr_default_torch_uses_ppocrv6(monkeypatch, tmp_path: Path) -> None: params, downloaded = _build( monkeypatch, - RapidOcrOptions(backend="torch"), # default lang -> chinese -> ch -> v6 + RapidOcrOptions(backend="torch"), # default lang -> zh-Hans -> ch -> v6 tmp_path, - seed=("torch", "chinese"), + seed=("torch", "zh-Hans"), ) assert Path(params["Det.model_path"]).name == "PP-OCRv6_det_small.pth" assert Path(params["Rec.model_path"]).name == "PP-OCRv6_rec_small.pth" @@ -193,23 +220,31 @@ def test_rapidocr_thai_uses_ppocrv5(monkeypatch, tmp_path: Path) -> None: def test_rapidocr_arabic_torch_uses_ppocrv4(monkeypatch, tmp_path: Path) -> None: params, _ = _build( monkeypatch, - RapidOcrOptions(lang=["arabic"], backend="torch"), + RapidOcrOptions(lang=["native:arabic"], backend="torch"), tmp_path, - seed=("torch", "arabic"), + seed=("torch", "native:arabic"), ) assert Path(params["Rec.model_path"]).name == "arabic_PP-OCRv4_rec_mobile.pth" # v4 rec ships a character dictionary. assert params["Rec.rec_keys_path"] is not None +def test_rapidocr_malformed_language_raises_at_options_time() -> None: + """A typo never reaches the model: the options validator rejects it.""" + with pytest.raises(ValidationError, match="BCP-47"): + RapidOcrOptions(lang=["klingon"], backend="onnxruntime") + + def test_rapidocr_unsupported_language_raises(monkeypatch, tmp_path: Path) -> None: captured_params: list[dict[str, object]] = [] _install_fakes(monkeypatch, captured_params) - with pytest.raises(ValueError): + # Georgian must be spelled out: a bare `ka` given to RapidOCR is PP-OCR's own + # code for Kannada. + with pytest.raises(OcrLanguageNotSupportedError, match="ka-Geor"): RapidOcrModel( enabled=True, artifacts_path=tmp_path, - options=RapidOcrOptions(lang=["klingon"], backend="onnxruntime"), + options=RapidOcrOptions(lang=["ka-Geor"], backend="onnxruntime"), accelerator_options=AcceleratorOptions(), ) @@ -309,7 +344,7 @@ def test_rapidocr_artifacts_missing_raises_with_prefetch_hint( assert "th_PP-OCRv5_rec_mobile.onnx" in message # The message must hand the user a command that actually fixes it. assert "docling-tools models download rapidocr" in message - assert "--rapidocr-backend-lang onnxruntime:th" in message + assert "--rapidocr-backend-lang onnxruntime:th-Thai" in message assert f"-o {tmp_path}" in message @@ -414,7 +449,7 @@ def fake_download_models(**kwargs: object) -> None: assert len(captured_calls) == 2 assert {call["backend"] for call in captured_calls} == {"torch", "onnxruntime"} # Both defaults resolve to PP-OCRv6, whose det/rec cover every v6 language. - assert {call["lang"] for call in captured_calls} == {"ch"} + assert {call["lang"] for call in captured_calls} == {"zh-Hans"} def test_model_downloader_rapidocr_models_replaces_default( @@ -460,18 +495,19 @@ def test_model_downloader_rejects_bad_rapidocr_spec(tmp_path: Path) -> None: @pytest.mark.parametrize( - "spec", ["onnxruntime:th", "torch:ka", "paddle:ch", "openvino:el"] + "spec", ["onnxruntime:th", "torch:kn", "paddle:zh-Hans", "openvino:el"] ) def test_parse_rapidocr_model_spec_accepts_valid_pairs(spec: str) -> None: parsed = _parse_rapidocr_model_spec(spec) assert f"{parsed.backend}:{parsed.user_lang}" == spec # Parsing yields the requested form only; resolution is left to the consumer. assert parsed.ppocr_version is None - assert parsed.rapidocr_lang_token is None + assert parsed.rapidocr_code is None @pytest.mark.parametrize( - "spec", ["torch:th", "torch:el", "onnxruntime:ka", "bogus:en", "no-colon", "a:b:c"] + "spec", + ["torch:th", "torch:el", "onnxruntime:ka-Geor", "bogus:en", "no-colon", "a:b:c"], ) def test_parse_rapidocr_model_spec_rejects_invalid_pairs(spec: str) -> None: with pytest.raises(ValueError): diff --git a/tests/test_tesseract_ocr_cli_lang.py b/tests/test_tesseract_ocr_cli_lang.py index fee4ded454..58cf6dbfac 100644 --- a/tests/test_tesseract_ocr_cli_lang.py +++ b/tests/test_tesseract_ocr_cli_lang.py @@ -24,18 +24,17 @@ def _model_for_listing(listing: str) -> TesseractOcrCliModel: f"{_MODULE}.subprocess.run", return_value=_FakeCompletedProcess(listing.encode("utf-8")), ): - model._set_languages_and_prefix() + model._set_languages() return model @pytest.mark.parametrize("sep", ["/", "\\"], ids=["posix", "windows"]) def test_script_packs_are_listed_with_either_separator(sep: str): - """Windows tesseract prints `script\\Arabic`; the prefix must still be detected.""" + """Windows tesseract prints `script\\Arabic`; the listing is normalized either way.""" model = _model_for_listing( f"List of available languages (3):\neng\nscript{sep}Arabic\nscript{sep}Latin\n" ) - assert model._script_prefix == "script/" - assert "script/Arabic" in model._tesseract_languages + assert "script/Arabic" in model._tesseract_vocabulary def test_detected_script_resolves_against_a_windows_listing(): diff --git a/tests/test_tesseract_ocr_lang.py b/tests/test_tesseract_ocr_lang.py new file mode 100644 index 0000000000..4f902072d0 --- /dev/null +++ b/tests/test_tesseract_ocr_lang.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT + +"""Tesseract's language coverage, checked against the real installation. + +No mocks: the installed tessdata set is read from the binary, and the assertions +are phrased against whatever that set turns out to be. +""" + +import shutil +import subprocess + +import pytest + +from docling.datamodel.accelerator_options import AcceleratorOptions +from docling.datamodel.pipeline_options import TesseractCliOcrOptions +from docling.exceptions import OcrLanguageNotSupportedError +from docling.models.stages.ocr.tesseract_ocr_cli_model import TesseractOcrCliModel +from docling.models.stages.ocr.tesseract_utils import installed_tesseract_tags + +pytestmark = pytest.mark.ml_ocr + + +def _installed_languages() -> list[str]: + if shutil.which("tesseract") is None: + pytest.skip("tesseract binary not installed") + output = subprocess.run( + ["tesseract", "--list-langs"], capture_output=True, check=True + ) + return output.stdout.decode("utf-8").splitlines()[1:] + + +def _build(lang: list[str]) -> TesseractOcrCliModel: + return TesseractOcrCliModel( + enabled=True, + artifacts_path=None, + options=TesseractCliOcrOptions(lang=lang), + accelerator_options=AcceleratorOptions(), + ) + + +def test_installed_language_maps_to_its_traineddata_name() -> None: + installed = _installed_languages() + if "eng" not in installed: + pytest.skip("the eng traineddata is not installed") + + model = _build(["en"]) + + assert model._native_codes == ["eng"] + + +def test_uninstalled_language_fails_at_construction() -> None: + """Tesseract never validated `options.lang` before, so a missing traineddata + surfaced as a per-page CLI failure much later.""" + installed = _installed_languages() + # Pick a language whose traineddata is definitely absent. + candidates = [("ka", "kat"), ("th", "tha"), ("el", "ell"), ("hi", "hin")] + choice = next( + (tag for tag, name in candidates if name not in installed), + None, + ) + if choice is None: + pytest.skip("every candidate language is installed") + + with pytest.raises(OcrLanguageNotSupportedError) as excinfo: + _build([choice]) + + message = str(excinfo.value) + assert "Supported:" in message + # The message names the installed set, as canonical tags. + if "eng" in installed: + assert "en-Latn" in message + + +def test_empty_lang_requires_the_osd_traineddata() -> None: + """An empty list runs orientation-and-script detection, which needs its own + file. No language is resolved up front: OSD picks one per page.""" + installed = _installed_languages() + if "osd" in installed: + model = _build([]) + assert model._auto_script is True + assert model._native_codes == [] + else: + with pytest.raises(ImportError, match="osd"): + _build([]) + + +def test_language_order_is_preserved_for_the_plus_join() -> None: + """Tesseract treats `-l a+b` order as preference order.""" + installed = _installed_languages() + if "eng" not in installed or "osd" not in installed: + pytest.skip("needs both eng and osd installed") + + model = _build(["en", "en-US", "eng"]) + + # Duplicates collapse; a single language remains. + assert model._native_codes == ["eng"] + + +def test_unprefixed_script_traineddata_is_advertised_natively() -> None: + """Some tessdata installs list a hand-placed script pack without the prefix. + + Only `script/` selects a pack through the OSD path, so a bare file is + named back verbatim behind `native:`: that is the only spelling that reaches + Tesseract unchanged. Left as a plain tag, `Latin` is unparseable and `Lao` + would read as the Lao language, whose `lao` traineddata is not installed. + """ + names = ["eng", "Latin", "Cyrillic", "Lao", "Japanese_vert"] + + tags = installed_tesseract_tags(names) + + assert tags == [ + "en-Latn", + "native:Cyrillic", + "native:Japanese_vert", + "native:Lao", + "native:Latin", + ] diff --git a/uv.lock b/uv.lock index 533ab095fd..1d7fa21bf4 100644 --- a/uv.lock +++ b/uv.lock @@ -881,7 +881,7 @@ name = "coloredlogs" version = "15.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "humanfriendly" }, + { name = "humanfriendly", marker = "python_full_version < '3.14'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520, upload-time = "2021-06-11T10:22:45.202Z" } wheels = [ @@ -918,7 +918,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/54/eb9bfc647b19f2009dd5c7f5ec51c4e6ca831725f1aea7a993034f483147/contourpy-1.3.2.tar.gz", hash = "sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54", size = 13466130, upload-time = "2025-04-15T17:47:53.79Z" } wheels = [ @@ -1004,7 +1004,7 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } @@ -1220,8 +1220,8 @@ name = "cryptography" version = "50.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "cffi", marker = "(python_full_version < '3.11' and platform_python_implementation != 'PyPy' and sys_platform == 'emscripten') or (python_full_version < '3.11' and platform_python_implementation != 'PyPy' and sys_platform == 'win32') or (platform_python_implementation != 'PyPy' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' and sys_platform != 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/de/41/6cbdcf9142d00fe82836fbb51e503e58088575cf7a0fe1dbff6695bf0840/cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9", size = 880201, upload-time = "2026-07-31T14:25:10.11Z" } wheels = [ @@ -1314,7 +1314,7 @@ name = "cuda-bindings" version = "13.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a9/21/8464d133752951c154feafb3b65c297e7d80f301183d220bec4c830f1441/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86", size = 6073403, upload-time = "2026-05-29T23:11:36.22Z" }, @@ -1352,37 +1352,37 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", version = "13.1.0.3", source = { registry = "https://pypi.org/simple" } }, + { name = "nvidia-cublas", version = "13.1.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] cudart = [ - { name = "nvidia-cuda-runtime" }, + { name = "nvidia-cuda-runtime", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] cufft = [ - { name = "nvidia-cufft" }, + { name = "nvidia-cufft", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] cufile = [ - { name = "nvidia-cufile" }, + { name = "nvidia-cufile", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] cupti = [ - { name = "nvidia-cuda-cupti" }, + { name = "nvidia-cuda-cupti", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] curand = [ - { name = "nvidia-curand" }, + { name = "nvidia-curand", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] cusolver = [ - { name = "nvidia-cusolver" }, + { name = "nvidia-cusolver", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] cusparse = [ - { name = "nvidia-cusparse" }, + { name = "nvidia-cusparse", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] nvtx = [ - { name = "nvidia-nvtx" }, + { name = "nvidia-nvtx", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] [[package]] @@ -1402,43 +1402,43 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", version = "13.1.1.3", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cublas", version = "13.1.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, + { name = "nvidia-cuda-nvrtc", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-runtime", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, ] cufft = [ - { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cufft", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, ] cufile = [ - { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cufile", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-cupti", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, ] curand = [ - { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-curand", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, ] cusolver = [ - { name = "nvidia-cublas", version = "13.1.1.3", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cublas", version = "13.1.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, + { name = "nvidia-cusolver", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, + { name = "nvidia-cusparse", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvtx", marker = "(python_full_version != '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux')" }, ] [[package]] @@ -1834,6 +1834,7 @@ dependencies = [ { name = "certifi" }, { name = "docling-core" }, { name = "filetype" }, + { name = "langcodes" }, { name = "pluggy" }, { name = "pydantic" }, { name = "pydantic-settings" }, @@ -2207,6 +2208,7 @@ requires-dist = [ { name = "filetype", specifier = ">=1.2.0,<2.0.0" }, { name = "httpx", marker = "extra == 'service-client'", specifier = ">=0.28,<1.0.0" }, { name = "huggingface-hub", marker = "extra == 'models-local'", specifier = ">=0.23,<2" }, + { name = "langcodes", specifier = ">=3.5.0,<4.0.0" }, { name = "librosa", marker = "extra == 'format-video'", specifier = ">=0.10.0,<1.0.0" }, { name = "lxml", marker = "extra == 'format-xml-jats'", specifier = ">=4.0.0,<7.0.0" }, { name = "mail-parser", marker = "extra == 'format-email'", specifier = ">=4.1.4,<5.0.0" }, @@ -2417,7 +2419,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -2448,9 +2450,9 @@ version = "1.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "packaging" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*' and sys_platform != 'win32'" }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform != 'win32'" }, + { name = "packaging", marker = "python_full_version < '3.11' or sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/59/68/20e91694ad9a8b2bb48af956899e52b645cb1501e7e2ec31cb733da4d4c5/faiss_cpu-1.15.0-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:50ea471ef1f4f3580eda8ab0ec9727d4bf65fd71c444bf306ce7cdbba8a42b21", size = 4904897, upload-time = "2026-08-03T17:49:37.003Z" }, @@ -2757,13 +2759,13 @@ name = "gliner" version = "0.2.28" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub" }, - { name = "onnxruntime" }, - { name = "sentencepiece" }, + { name = "huggingface-hub", marker = "python_full_version < '3.14'" }, + { name = "onnxruntime", marker = "python_full_version < '3.14'" }, + { name = "sentencepiece", marker = "python_full_version < '3.14'" }, { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, - { name = "tqdm" }, - { name = "transformers" }, + { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.14' and platform_machine != 'x86_64') or (python_full_version < '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.14' and sys_platform != 'linux')" }, + { name = "tqdm", marker = "python_full_version < '3.14'" }, + { name = "transformers", marker = "python_full_version < '3.14'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f4/b7/0f3e24ff0b8c1c95121532a44e09b5bb87bd771c6bed0d387d51480645c5/gliner-0.2.28.tar.gz", hash = "sha256:b1637afb5cf4235fc871f1e21498831775b7bd19cefbda6dc5fe08ee88cb07a0", size = 263394, upload-time = "2026-07-24T14:03:49.833Z" } wheels = [ @@ -3024,7 +3026,7 @@ name = "humanfriendly" version = "10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyreadline3", marker = "sys_platform == 'win32'" }, + { name = "pyreadline3", marker = "python_full_version < '3.14' and sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } wheels = [ @@ -3119,17 +3121,17 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "decorator" }, - { name = "exceptiongroup" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, - { name = "typing-extensions" }, + { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, + { name = "decorator", marker = "python_full_version < '3.11'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "jedi", marker = "python_full_version < '3.11'" }, + { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, + { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, + { name = "pygments", marker = "python_full_version < '3.11'" }, + { name = "stack-data", marker = "python_full_version < '3.11'" }, + { name = "traitlets", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } wheels = [ @@ -3160,17 +3162,17 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "ipython-pygments-lexers" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, + { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, + { name = "jedi", marker = "python_full_version >= '3.11'" }, + { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, + { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, + { name = "psutil", marker = "python_full_version >= '3.11' and sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, + { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "stack-data", marker = "python_full_version >= '3.11'" }, + { name = "traitlets", marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/96/b150fe7e25a5a29ae9ac1374e71488639605d39a1ea4abb74c9ce33af235/ipython-9.16.1.tar.gz", hash = "sha256:5a3d1f9a47ff216d6cf9cf863124f6a2c1a198d1354c546a4d24a370a283b64c", size = 4515302, upload-time = "2026-08-03T08:36:15.571Z" } wheels = [ @@ -3182,7 +3184,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments" }, + { name = "pygments", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -3652,6 +3654,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d3/26/1ef06f56198d631296d646a6223de35bcc6cf9795ceb2442816bc963b84c/langchain_text_splitters-1.1.2-py3-none-any.whl", hash = "sha256:a2de0d799ff31886429fd6e2e0032df275b60ec817c19059a7b46181cc1c2f10", size = 35903, upload-time = "2026-04-16T14:20:38.243Z" }, ] +[[package]] +name = "langcodes" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/75/f9edc5d72945019312f359e69ded9f82392a81d49c5051ed3209b100c0d2/langcodes-3.5.1.tar.gz", hash = "sha256:40bff315e01b01d11c2ae3928dd4f5cbd74dd38f9bd912c12b9a3606c143f731", size = 191084, upload-time = "2025-12-02T16:22:01.627Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/c1/d10b371bcba7abce05e2b33910e39c33cfa496a53f13640b7b8e10bb4d2b/langcodes-3.5.1-py3-none-any.whl", hash = "sha256:b6a9c25c603804e2d169165091d0cdb23934610524a21d226e4f463e8e958a72", size = 183050, upload-time = "2025-12-02T16:21:59.954Z" }, +] + [[package]] name = "langsmith" version = "0.11.0" @@ -4059,15 +4070,15 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "contourpy", version = "1.3.2", source = { registry = "https://pypi.org/simple" } }, - { name = "cycler" }, - { name = "fonttools" }, - { name = "kiwisolver" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "packaging" }, - { name = "pillow" }, - { name = "pyparsing" }, - { name = "python-dateutil" }, + { name = "contourpy", version = "1.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "cycler", marker = "python_full_version < '3.11'" }, + { name = "fonttools", marker = "python_full_version < '3.11'" }, + { name = "kiwisolver", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "packaging", marker = "python_full_version < '3.11'" }, + { name = "pillow", marker = "python_full_version < '3.11'" }, + { name = "pyparsing", marker = "python_full_version < '3.11'" }, + { name = "python-dateutil", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/63/1b/4be5be87d43d327a0cf4de1a56e86f7f84c89312452406cf122efe2839e6/matplotlib-3.10.9.tar.gz", hash = "sha256:fd66508e8c6877d98e586654b608a0456db8d7e8a546eb1e2600efd957302358", size = 34811233, upload-time = "2026-04-24T00:14:13.539Z" } wheels = [ @@ -4151,16 +4162,16 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "contourpy", version = "1.3.3", source = { registry = "https://pypi.org/simple" } }, - { name = "cycler" }, - { name = "fonttools" }, - { name = "kiwisolver" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "contourpy", version = "1.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "cycler", marker = "python_full_version >= '3.11'" }, + { name = "fonttools", marker = "python_full_version >= '3.11'" }, + { name = "kiwisolver", marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "pyparsing" }, - { name = "python-dateutil" }, + { name = "packaging", marker = "python_full_version >= '3.11'" }, + { name = "pillow", marker = "python_full_version >= '3.11'" }, + { name = "pyparsing", marker = "python_full_version >= '3.11'" }, + { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/49/64/f9a391af28f518b11ad45a8a712353c94a0aefce09d3703200e5c54b610a/matplotlib-3.11.1.tar.gz", hash = "sha256:69647db5746941c793d6e445a4cd349323ffb87d9cc958c2ad84a659b4832d30", size = 32612045, upload-time = "2026-07-18T03:39:46.63Z" } wheels = [ @@ -4258,12 +4269,12 @@ name = "milvus-lite" version = "3.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "faiss-cpu" }, - { name = "grpcio" }, + { name = "faiss-cpu", marker = "python_full_version < '3.11' or sys_platform != 'win32'" }, + { name = "grpcio", marker = "python_full_version < '3.11' or sys_platform != 'win32'" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "pyarrow" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*' and sys_platform != 'win32'" }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform != 'win32'" }, + { name = "pyarrow", marker = "python_full_version < '3.11' or sys_platform != 'win32'" }, { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/59/caab693e260fb00704672838f332eb0ffeff552806d270ee61f736e14dd4/milvus_lite-3.2.0.tar.gz", hash = "sha256:d6735d7a5bcb14fed2cc9a42da84d7396fef0040a0f7c2d1cace9f2134925571", size = 719788, upload-time = "2026-08-06T08:58:39.198Z" } @@ -4276,7 +4287,7 @@ name = "miniaudio" version = "1.71" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi" }, + { name = "cffi", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d8/d5/e5439dc08561f73656bfeb3340fc64ab63163e101426593d8fb9a025ff1e/miniaudio-1.71.tar.gz", hash = "sha256:ff51e2887bb673e2e757752b586b3dc924d59aa5fbcae9bbc45f4a111bd3262b", size = 1116480, upload-time = "2026-04-29T21:20:38.182Z" } wheels = [ @@ -4453,7 +4464,7 @@ name = "mlx" version = "0.32.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mlx-metal" }, + { name = "mlx-metal", marker = "sys_platform == 'darwin'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a5/f1/78eb4402dc425da28c329cfd7570cd8314fe9d000010c41186c018748f07/mlx-0.32.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:9ffc013f58cca13b71bbc7615a88481f8f4352a5f03b083e9bb798d1f6aac374", size = 622966, upload-time = "2026-08-18T02:44:52.92Z" }, @@ -4484,19 +4495,19 @@ name = "mlx-audio" version = "0.4.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub" }, - { name = "miniaudio" }, - { name = "mlx" }, - { name = "mlx-lm" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "sounddevice" }, - { name = "tqdm" }, - { name = "transformers" }, + { name = "huggingface-hub", marker = "sys_platform == 'darwin'" }, + { name = "miniaudio", marker = "sys_platform == 'darwin'" }, + { name = "mlx", marker = "sys_platform == 'darwin'" }, + { name = "mlx-lm", marker = "sys_platform == 'darwin'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform == 'darwin'" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*' and sys_platform == 'darwin'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform == 'darwin'" }, + { name = "sounddevice", marker = "sys_platform == 'darwin'" }, + { name = "tqdm", marker = "sys_platform == 'darwin'" }, + { name = "transformers", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e7/02/40b042713edf6f8f7714f711a2fc5191b871c4c39d79df6e7726de6a81e9/mlx_audio-0.4.6.tar.gz", hash = "sha256:9f377ba4c0927af06526ed2d03b2fa44eb158d83385da96422da17c926b8589f", size = 1488412, upload-time = "2026-07-25T09:07:07.729Z" } wheels = [ @@ -4508,15 +4519,15 @@ name = "mlx-lm" version = "0.31.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jinja2" }, - { name = "mlx" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "protobuf" }, - { name = "pyyaml" }, - { name = "sentencepiece" }, - { name = "transformers" }, + { name = "jinja2", marker = "sys_platform == 'darwin'" }, + { name = "mlx", marker = "sys_platform == 'darwin'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform == 'darwin'" }, + { name = "protobuf", marker = "sys_platform == 'darwin'" }, + { name = "pyyaml", marker = "sys_platform == 'darwin'" }, + { name = "sentencepiece", marker = "sys_platform == 'darwin'" }, + { name = "transformers", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/84/94/9a38d6b0c6fcca995b9136c94eb7da1e9c5165652edf228b96b29960fa7a/mlx_lm-0.31.3.tar.gz", hash = "sha256:61eb0e3ba09444f77f874aff295401d7ccd20b39495cbbce0c782a15474ce733", size = 304318, upload-time = "2026-04-22T07:37:27.922Z" } wheels = [ @@ -4538,24 +4549,24 @@ name = "mlx-vlm" version = "0.6.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "datasets" }, - { name = "fastapi" }, - { name = "llguidance" }, - { name = "miniaudio" }, - { name = "mlx" }, - { name = "mlx-audio" }, - { name = "mlx-lm" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "opencv-python" }, - { name = "pillow" }, - { name = "python-multipart" }, - { name = "requests" }, - { name = "starlette" }, - { name = "tqdm" }, - { name = "transformers" }, - { name = "uvicorn" }, + { name = "datasets", marker = "sys_platform == 'darwin'" }, + { name = "fastapi", marker = "sys_platform == 'darwin'" }, + { name = "llguidance", marker = "sys_platform == 'darwin'" }, + { name = "miniaudio", marker = "sys_platform == 'darwin'" }, + { name = "mlx", marker = "sys_platform == 'darwin'" }, + { name = "mlx-audio", marker = "sys_platform == 'darwin'" }, + { name = "mlx-lm", marker = "sys_platform == 'darwin'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform == 'darwin'" }, + { name = "opencv-python", marker = "sys_platform == 'darwin'" }, + { name = "pillow", marker = "sys_platform == 'darwin'" }, + { name = "python-multipart", marker = "sys_platform == 'darwin'" }, + { name = "requests", marker = "sys_platform == 'darwin'" }, + { name = "starlette", marker = "sys_platform == 'darwin'" }, + { name = "tqdm", marker = "sys_platform == 'darwin'" }, + { name = "transformers", marker = "sys_platform == 'darwin'" }, + { name = "uvicorn", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/de/03810d375be44e04a0889a7709aa72d8f187ec94a5172dea3d91051032e4/mlx_vlm-0.6.4.tar.gz", hash = "sha256:2a911692aedc3861ae26f4057b1c05dcb9abfb954d50123df3ef63eab0c58e29", size = 1453442, upload-time = "2026-07-06T21:11:12.567Z" } wheels = [ @@ -4567,19 +4578,19 @@ name = "mlx-whisper" version = "0.4.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub" }, - { name = "mlx" }, - { name = "more-itertools" }, - { name = "numba" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "tiktoken" }, - { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, - { name = "tqdm" }, + { name = "huggingface-hub", marker = "sys_platform == 'darwin'" }, + { name = "mlx", marker = "sys_platform == 'darwin'" }, + { name = "more-itertools", marker = "sys_platform == 'darwin'" }, + { name = "numba", marker = "sys_platform == 'darwin'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform == 'darwin'" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*' and sys_platform == 'darwin'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform == 'darwin'" }, + { name = "tiktoken", marker = "sys_platform == 'darwin'" }, + { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "tqdm", marker = "sys_platform == 'darwin'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/22/b7/a35232812a2ccfffcb7614ba96a91338551a660a0e9815cee668bf5743f0/mlx_whisper-0.4.3-py3-none-any.whl", hash = "sha256:6b82b6597a994643a3e5496c7bc229a672e5ca308458455bfe276e76ae024489", size = 890544, upload-time = "2025-08-29T14:56:13.815Z" }, @@ -4970,12 +4981,12 @@ name = "nemotron-ocr" version = "2.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub" }, - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" } }, - { name = "pillow" }, - { name = "shapely" }, - { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" } }, - { name = "torchvision", version = "0.26.0", source = { registry = "https://pypi.org/simple" } }, + { name = "huggingface-hub", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "pillow", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "shapely", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "torchvision", version = "0.26.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/8e/42/744f74d37f58c4589bda4b8c3b0477432c48ff5caa3f1e1050d8c0edafff/nemotron_ocr-2.0.2.tar.gz", hash = "sha256:703ae32bf7172da8985fca90ff22584fd9622ee04ce877d0cc6d2db4be7dce5b", size = 159827, upload-time = "2026-07-20T19:05:24.126Z" } wheels = [ @@ -5395,7 +5406,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, @@ -5437,7 +5448,7 @@ resolution-markers = [ "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", ] dependencies = [ - { name = "nvidia-cublas", version = "13.1.0.3", source = { registry = "https://pypi.org/simple" } }, + { name = "nvidia-cublas", version = "13.1.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a3/22/0b4b932655d17a6da1b92fa92ab12844b053bb2ac2475e179ba6f043da1e/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:d20e1734305e9d68889a96e3f35094d733ff1f83932ebe462753973e53a572bf", size = 366066321, upload-time = "2026-02-03T20:44:52.837Z" }, @@ -5455,7 +5466,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "nvidia-cublas", version = "13.1.1.3", source = { registry = "https://pypi.org/simple" } }, + { name = "nvidia-cublas", version = "13.1.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, @@ -5467,7 +5478,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -5497,10 +5508,10 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", version = "13.1.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64'" }, - { name = "nvidia-cublas", version = "13.1.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64'" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, + { name = "nvidia-cublas", version = "13.1.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cublas", version = "13.1.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -5512,7 +5523,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -5605,9 +5616,9 @@ name = "ocrmac" version = "1.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "pillow" }, - { name = "pyobjc-framework-vision" }, + { name = "click", marker = "sys_platform == 'darwin'" }, + { name = "pillow", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-vision", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5e/07/3e15ab404f75875c5e48c47163300eb90b7409044d8711fc3aaf52503f2e/ocrmac-1.0.1.tar.gz", hash = "sha256:507fe5e4cbd67b2d03f6729a52bbc11f9d0b58241134eb958a5daafd4b9d93d9", size = 1454317, upload-time = "2026-01-08T16:44:26.412Z" } wheels = [ @@ -5654,14 +5665,14 @@ name = "onnxruntime" version = "1.23.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "coloredlogs" }, - { name = "flatbuffers" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, + { name = "coloredlogs", marker = "python_full_version < '3.14'" }, + { name = "flatbuffers", marker = "python_full_version < '3.14'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and python_full_version < '3.14'" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "sympy" }, + { name = "packaging", marker = "python_full_version < '3.14'" }, + { name = "protobuf", marker = "python_full_version < '3.14'" }, + { name = "sympy", marker = "python_full_version < '3.14'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/35/d6/311b1afea060015b56c742f3531168c1644650767f27ef40062569960587/onnxruntime-1.23.2-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:a7730122afe186a784660f6ec5807138bf9d792fa1df76556b27307ea9ebcbe3", size = 17195934, upload-time = "2025-10-27T23:06:14.143Z" }, @@ -5693,14 +5704,14 @@ name = "onnxruntime-gpu" version = "1.23.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "coloredlogs" }, - { name = "flatbuffers" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and python_full_version < '3.14'" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "sympy" }, + { name = "coloredlogs", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten')" }, + { name = "flatbuffers", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform != 'darwin'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten'" }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten'" }, + { name = "packaging", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten')" }, + { name = "protobuf", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten')" }, + { name = "sympy", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/cb/ae/39283748c68a96be4f5f8a9561e0e3ca92af1eae6c2b1c07fb1da5f65cd1/onnxruntime_gpu-1.23.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18de50c6c8eea50acc405ea13d299aec593e46478d7a22cd32cdbbdf7c42899d", size = 300525411, upload-time = "2025-10-22T16:56:08.415Z" }, @@ -5877,10 +5888,10 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "python-dateutil" }, - { name = "pytz" }, - { name = "tzdata" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "python-dateutil", marker = "python_full_version < '3.11'" }, + { name = "pytz", marker = "python_full_version < '3.11'" }, + { name = "tzdata", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -5957,10 +5968,10 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "python-dateutil" }, - { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, + { name = "tzdata", marker = "(python_full_version >= '3.11' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } wheels = [ @@ -6078,7 +6089,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -6852,7 +6863,7 @@ name = "pyobjc-framework-cocoa" version = "12.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/75/76/49c6da2c6a831020b4854ba20079d5a1030474bffc776b7b73c2eeff8c15/pyobjc_framework_cocoa-12.2.2.tar.gz", hash = "sha256:c96c0ef69a71afbbb0e6a7d594b455c5fe47d62e0db376ee7a2b4b828c16ace9", size = 3132831, upload-time = "2026-08-11T19:44:02.288Z" } wheels = [ @@ -6872,8 +6883,8 @@ name = "pyobjc-framework-coreml" version = "12.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, - { name = "pyobjc-framework-cocoa" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b7/3b/c835535e7aef41afc953e5597ea41e693b7bae80d753ab74b9721e07cba5/pyobjc_framework_coreml-12.2.2.tar.gz", hash = "sha256:3e6abe134634adbcc0c1e843826e42df86e63beb3bc7e8e4a914e93179ae1e75", size = 49750, upload-time = "2026-08-11T19:44:14.011Z" } wheels = [ @@ -6893,8 +6904,8 @@ name = "pyobjc-framework-quartz" version = "12.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, - { name = "pyobjc-framework-cocoa" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/35/b1/426a37c7ae37280b3ffca2571fb48f211946aee2f4ca31a603ed1943c4a7/pyobjc_framework_quartz-12.2.2.tar.gz", hash = "sha256:810f97b210cfd93704d240860286dfd6df09f9f1c52525fc5c2166723aea3f9e", size = 3218295, upload-time = "2026-08-11T19:45:15.189Z" } wheels = [ @@ -6914,10 +6925,10 @@ name = "pyobjc-framework-vision" version = "12.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, - { name = "pyobjc-framework-cocoa" }, - { name = "pyobjc-framework-coreml" }, - { name = "pyobjc-framework-quartz" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-coreml", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ea/bf/31e8bcae94047b365592ab7533096a4025c0306a8a312b65bcbf57e073ec/pyobjc_framework_vision-12.2.2.tar.gz", hash = "sha256:a6bf78e8dca145c6a78fd8d5f925b6da54649a60a1464b81ff405bca4a08406b", size = 73289, upload-time = "2026-08-11T19:45:41.877Z" } wheels = [ @@ -8130,14 +8141,14 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "imageio" }, - { name = "lazy-loader" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" } }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "packaging" }, - { name = "pillow" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, - { name = "tifffile", version = "2025.5.10", source = { registry = "https://pypi.org/simple" } }, + { name = "imageio", marker = "python_full_version < '3.11'" }, + { name = "lazy-loader", marker = "python_full_version < '3.11'" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "packaging", marker = "python_full_version < '3.11'" }, + { name = "pillow", marker = "python_full_version < '3.11'" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "tifffile", version = "2025.5.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c7/a8/3c0f256012b93dd2cb6fda9245e9f4bff7dc0486880b248005f15ea2255e/scikit_image-0.25.2.tar.gz", hash = "sha256:e5a37e6cd4d0c018a7a55b9d601357e3382826d3888c10d0213fc63bff977dde", size = 22693594, upload-time = "2025-02-18T18:05:24.538Z" } wheels = [ @@ -8188,16 +8199,16 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "imageio" }, - { name = "lazy-loader" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" } }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "imageio", marker = "python_full_version >= '3.11'" }, + { name = "lazy-loader", marker = "python_full_version >= '3.11'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "packaging", marker = "python_full_version >= '3.11'" }, + { name = "pillow", marker = "python_full_version >= '3.11'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "tifffile", version = "2026.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "tifffile", version = "2026.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "tifffile", version = "2026.8.16", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a1/b4/2528bb43c67d48053a7a649a9666432dc307d66ba02e3a6d5c40f46655df/scikit_image-0.26.0.tar.gz", hash = "sha256:f5f970ab04efad85c24714321fcc91613fcb64ef2a892a13167df2f3e59199fa", size = 22729739, upload-time = "2025-12-20T17:12:21.824Z" } @@ -8261,10 +8272,10 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "joblib" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, - { name = "threadpoolctl" }, + { name = "joblib", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "threadpoolctl", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/c2/a7855e41c9d285dfe86dc50b250978105dce513d6e459ea66a6aeb0e1e0c/scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda", size = 7193136, upload-time = "2025-09-09T08:21:29.075Z" } wheels = [ @@ -8324,13 +8335,13 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "joblib" }, - { name = "narwhals" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "joblib", marker = "python_full_version >= '3.11'" }, + { name = "narwhals", marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "threadpoolctl" }, + { name = "threadpoolctl", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } wheels = [ @@ -8375,7 +8386,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -8437,7 +8448,7 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -8523,7 +8534,7 @@ resolution-markers = [ "(python_full_version == '3.12.*' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", ] dependencies = [ - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [ @@ -8574,8 +8585,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography" }, - { name = "jeepney" }, + { name = "cryptography", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "jeepney", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -8812,7 +8823,7 @@ name = "sounddevice" version = "0.5.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi" }, + { name = "cffi", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ec/db/0c890e2d9aab9ba284021efc02e1d3aebfecab1b611762d7434602209bcf/sounddevice-0.5.6.tar.gz", hash = "sha256:8ec9fbfde2e32f020b167e348f3ab3bac6625a5f15af524d790108ac7147a410", size = 1120094, upload-time = "2026-08-17T07:55:05.048Z" } wheels = [ @@ -8904,8 +8915,8 @@ name = "standard-aifc" version = "3.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "audioop-lts" }, - { name = "standard-chunk" }, + { name = "audioop-lts", marker = "python_full_version >= '3.13'" }, + { name = "standard-chunk", marker = "python_full_version >= '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c4/53/6050dc3dde1671eb3db592c13b55a8005e5040131f7509cef0215212cb84/standard_aifc-3.13.0.tar.gz", hash = "sha256:64e249c7cb4b3daf2fdba4e95721f811bde8bdfc43ad9f936589b7bb2fae2e43", size = 15240, upload-time = "2024-10-30T16:01:31.772Z" } wheels = [ @@ -8926,7 +8937,7 @@ name = "standard-sunau" version = "3.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "audioop-lts" }, + { name = "audioop-lts", marker = "python_full_version >= '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/e3/ce8d38cb2d70e05ffeddc28bb09bad77cfef979eb0a299c9117f7ed4e6a9/standard_sunau-3.13.0.tar.gz", hash = "sha256:b319a1ac95a09a2378a8442f403c66f4fd4b36616d6df6ae82b8e536ee790908", size = 9368, upload-time = "2024-10-30T16:01:41.626Z" } wheels = [ @@ -9070,7 +9081,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/44/d0/18fed0fc0916578a4463f775b0fbd9c5fed2392152d039df2fb533bfdd5d/tifffile-2025.5.10.tar.gz", hash = "sha256:018335d34283aa3fd8c263bae5c3c2b661ebc45548fde31504016fcae7bf1103", size = 365290, upload-time = "2025-05-10T19:22:34.386Z" } wheels = [ @@ -9088,7 +9099,7 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c5/cb/2f6d79c7576e22c116352a801f4c3c8ace5957e9aced862012430b62e14f/tifffile-2026.3.3.tar.gz", hash = "sha256:d9a1266bed6f2ee1dd0abde2018a38b4f8b2935cb843df381d70ac4eac5458b7", size = 388745, upload-time = "2026-03-03T19:14:38.134Z" } wheels = [ @@ -9115,7 +9126,7 @@ resolution-markers = [ "(python_full_version == '3.12.*' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", ] dependencies = [ - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4b/22/5de8a60ec7cd23436a9a5e2c5be8866556796307e0f54e0fbc2bf1981677/tifffile-2026.8.16.tar.gz", hash = "sha256:523a18b8183253e1e594162eed45f157c959d8cff6decde3f2d58f20a5a45389", size = 442370, upload-time = "2026-08-15T16:09:35.736Z" } wheels = [ @@ -9321,20 +9332,20 @@ resolution-markers = [ "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", ] dependencies = [ - { name = "cuda-bindings" }, - { name = "cuda-toolkit", version = "13.0.2", source = { registry = "https://pypi.org/simple" }, extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"] }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" } }, - { name = "nvidia-cudnn-cu13", version = "9.19.0.56", source = { registry = "https://pypi.org/simple" } }, - { name = "nvidia-cusparselt-cu13", version = "0.8.0", source = { registry = "https://pypi.org/simple" } }, - { name = "nvidia-nccl-cu13", version = "2.28.9", source = { registry = "https://pypi.org/simple" } }, - { name = "nvidia-nvshmem-cu13" }, - { name = "setuptools", version = "81.0.0", source = { registry = "https://pypi.org/simple" } }, - { name = "sympy" }, - { name = "triton", version = "3.6.0", source = { registry = "https://pypi.org/simple" } }, - { name = "typing-extensions" }, + { name = "cuda-bindings", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "cuda-toolkit", version = "13.0.2", source = { registry = "https://pypi.org/simple" }, extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "filelock", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "fsspec", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "jinja2", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cudnn-cu13", version = "9.19.0.56", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu13", version = "0.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu13", version = "2.28.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu13", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "setuptools", version = "81.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "sympy", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "triton", version = "3.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f9/1e/18a9b10b4bd34f12d4e561c52b0ae7158707b8193c6cfc0aad2b48167090/torch-2.11.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:1b32ceda909818a03b112006709b02be1877240c31750a8d9c6b7bf5f2d8a6e5", size = 530589207, upload-time = "2026-03-23T18:11:23.756Z" }, @@ -9371,21 +9382,21 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "cuda-bindings", marker = "python_full_version < '3.15' and sys_platform == 'linux'" }, - { name = "cuda-toolkit", version = "13.0.3.0", source = { registry = "https://pypi.org/simple" }, extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, + { name = "cuda-bindings", marker = "(python_full_version >= '3.13' and python_full_version < '3.15' and sys_platform == 'linux') or (python_full_version < '3.15' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "cuda-toolkit", version = "13.0.3.0", source = { registry = "https://pypi.org/simple" }, extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "(python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux')" }, + { name = "filelock", marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "fsspec", marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "jinja2", marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and platform_machine != 'x86_64') or (python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.11' and sys_platform != 'linux')" }, - { name = "nvidia-cudnn-cu13", version = "9.20.0.48", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", version = "0.8.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", version = "2.29.7", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, - { name = "setuptools", version = "84.0.0", source = { registry = "https://pypi.org/simple" } }, - { name = "sympy" }, - { name = "triton", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15' and sys_platform == 'linux'" }, - { name = "typing-extensions" }, + { name = "nvidia-cudnn-cu13", version = "9.20.0.48", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusparselt-cu13", version = "0.8.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nccl-cu13", version = "2.29.7", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvshmem-cu13", marker = "(python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux')" }, + { name = "setuptools", version = "84.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "sympy", marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "triton", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and python_full_version < '3.15' and sys_platform == 'linux') or (python_full_version < '3.15' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/7f/e7/19894fdb51c7dbaf94f5a79bb0871da0992e8e4241e579cb006da46d2e58/torch-2.13.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:94f0de129916f77b8dc2c7a8eff644cfeddfe59e39c9f55e9f6e17543410281d", size = 111178962, upload-time = "2026-07-08T16:05:49.855Z" }, @@ -9422,9 +9433,9 @@ resolution-markers = [ "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", ] dependencies = [ - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" } }, - { name = "pillow" }, - { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "pillow", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/5e/00/24d8c7845c3f270153fb81395a5135b2778e2538e81d14c6aea5106c689c/torchvision-0.26.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b6f9ad1ecc0eab52647298b379ee9426845f8903703e6127973f8f3d049a798b", size = 7518249, upload-time = "2026-03-23T18:12:51.743Z" }, @@ -9464,8 +9475,8 @@ dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and platform_machine != 'x86_64') or (python_full_version >= '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and sys_platform != 'linux')" }, - { name = "pillow" }, - { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, + { name = "pillow", marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/b4/df/1ba039ad6cfe6e69209c36766b9b6e8c6fe92481c6d4e4ca52296f5f699d/torchvision-0.28.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:2a1ef4b6f4bf5828b48cfad97372c8982db906830884b2868ba5c3df937a7d81", size = 1856019, upload-time = "2026-07-08T16:07:59.283Z" },