From ed7154366e99705e17a5f0e6626f19a7ff2bf43e Mon Sep 17 00:00:00 2001 From: Nikos Livathinos Date: Fri, 21 Aug 2026 16:09:02 +0200 Subject: [PATCH 01/19] fix: Use a tmp dir in tests/test_conversion_result_json.py Signed-off-by: Nikos Livathinos --- tests/test_conversion_result_json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_conversion_result_json.py b/tests/test_conversion_result_json.py index e3855a10a8..10cb925346 100644 --- a/tests/test_conversion_result_json.py +++ b/tests/test_conversion_result_json.py @@ -13,7 +13,7 @@ from docling.document_converter import DocumentConverter, PdfFormatOption -def test_conversion_result_json_roundtrip_string(): +def test_conversion_result_json_roundtrip_string(tmp_path: Path): pdf_doc = Path("./tests/data/pdf/sources/redp5110_sampled.pdf") pipeline_options = PdfPipelineOptions() @@ -33,7 +33,7 @@ def test_conversion_result_json_roundtrip_string(): ) conv_res = doc_converter.convert(pdf_doc) - fpath: Path = Path("./test-conversion.zip") + fpath: Path = tmp_path / "test-conversion.zip" conv_res.save(filename=fpath) # returns string when no filename is given # assert isinstance(json_str, str) and len(json_str) > 0 From a7c9447c3ba035d2afa838e4a7a9702294ba870e Mon Sep 17 00:00:00 2001 From: Nikos Livathinos Date: Tue, 25 Aug 2026 18:55:32 +0200 Subject: [PATCH 02/19] chore: Introduce langcodes dependency Signed-off-by: Nikos Livathinos --- pyproject.toml | 1 + uv.lock | 499 +++++++++++++++++++++++++------------------------ 2 files changed, 256 insertions(+), 244 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cb67bf2cb0..ba015fa6f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ dependencies = [ 'certifi>=2024.7.4', 'pluggy>=1.0.0,<2.0.0', 'tqdm>=4.65.0,<5.0.0', + "langcodes>=3.5.0,<4.0.0", ] [project.urls] diff --git a/uv.lock b/uv.lock index fe8354bb16..3c113572f0 100644 --- a/uv.lock +++ b/uv.lock @@ -807,7 +807,7 @@ name = "coloredlogs" version = "15.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "humanfriendly" }, + { name = "humanfriendly", marker = "python_full_version < '3.14'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520, upload-time = "2021-06-11T10:22:45.202Z" } wheels = [ @@ -844,7 +844,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/54/eb9bfc647b19f2009dd5c7f5ec51c4e6ca831725f1aea7a993034f483147/contourpy-1.3.2.tar.gz", hash = "sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54", size = 13466130, upload-time = "2025-04-15T17:47:53.79Z" } wheels = [ @@ -930,7 +930,7 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } wheels = [ @@ -1115,8 +1115,8 @@ name = "cryptography" version = "49.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "cffi", marker = "(python_full_version < '3.11' and platform_python_implementation != 'PyPy' and sys_platform == 'emscripten') or (python_full_version < '3.11' and platform_python_implementation != 'PyPy' and sys_platform == 'win32') or (platform_python_implementation != 'PyPy' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' and sys_platform != 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1f/99/d1c90d6041656cc6ee229dc99cd67fd0cd5aec3c5f7d72fffc27cc750054/cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493", size = 854345, upload-time = "2026-06-12T20:02:30.512Z" } wheels = [ @@ -1207,7 +1207,7 @@ name = "cuda-bindings" version = "13.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a9/21/8464d133752951c154feafb3b65c297e7d80f301183d220bec4c830f1441/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86", size = 6073403, upload-time = "2026-05-29T23:11:36.22Z" }, @@ -1242,37 +1242,37 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] cudart = [ - { name = "nvidia-cuda-runtime" }, + { name = "nvidia-cuda-runtime", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, ] cufft = [ - { name = "nvidia-cufft" }, + { name = "nvidia-cufft", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, ] cufile = [ - { name = "nvidia-cufile" }, + { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, ] cupti = [ - { name = "nvidia-cuda-cupti" }, + { name = "nvidia-cuda-cupti", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, ] curand = [ - { name = "nvidia-curand" }, + { name = "nvidia-curand", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, ] cusolver = [ - { name = "nvidia-cusolver" }, + { name = "nvidia-cusolver", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, ] cusparse = [ - { name = "nvidia-cusparse" }, + { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, ] nvtx = [ - { name = "nvidia-nvtx" }, + { name = "nvidia-nvtx", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, ] [[package]] @@ -1667,6 +1667,7 @@ dependencies = [ { name = "certifi" }, { name = "docling-core" }, { name = "filetype" }, + { name = "langcodes" }, { name = "pluggy" }, { name = "pydantic" }, { name = "pydantic-settings" }, @@ -2036,6 +2037,7 @@ requires-dist = [ { name = "filetype", specifier = ">=1.2.0,<2.0.0" }, { name = "httpx", marker = "extra == 'service-client'", specifier = ">=0.28,<1.0.0" }, { name = "huggingface-hub", marker = "extra == 'models-local'", specifier = ">=0.23,<2" }, + { name = "langcodes", specifier = ">=3.5.0,<4.0.0" }, { name = "librosa", marker = "extra == 'format-video'", specifier = ">=0.10.0,<1.0.0" }, { name = "lxml", marker = "extra == 'format-xml-jats'", specifier = ">=4.0.0,<7.0.0" }, { name = "mail-parser", marker = "extra == 'format-email'", specifier = ">=4.1.4,<5.0.0" }, @@ -2243,7 +2245,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -2285,11 +2287,11 @@ name = "fastapi" version = "0.139.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "annotated-doc" }, - { name = "pydantic" }, - { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, + { name = "annotated-doc", marker = "sys_platform == 'darwin'" }, + { name = "pydantic", marker = "sys_platform == 'darwin'" }, + { name = "starlette", marker = "sys_platform == 'darwin'" }, + { name = "typing-extensions", marker = "sys_platform == 'darwin'" }, + { name = "typing-inspection", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d3/af/a5f50ccfa659ec1802cb4ca842c23f06d906a8cc9aef6016a2caeea3d4ed/fastapi-0.139.0.tar.gz", hash = "sha256:99ab7b2d92223c76d6cf10757ab3f89d45b38267fc20b2a136cf02f6beac3145", size = 423016, upload-time = "2026-07-01T16:35:33.436Z" } wheels = [ @@ -2564,13 +2566,13 @@ name = "gliner" version = "0.2.24" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub" }, - { name = "onnxruntime" }, - { name = "sentencepiece" }, + { name = "huggingface-hub", marker = "python_full_version < '3.14'" }, + { name = "onnxruntime", marker = "python_full_version < '3.14'" }, + { name = "sentencepiece", marker = "python_full_version < '3.14'" }, { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "torch", version = "2.12.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, - { name = "tqdm" }, - { name = "transformers" }, + { name = "torch", version = "2.12.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.14' and platform_machine != 'x86_64') or (python_full_version < '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.14' and sys_platform != 'linux')" }, + { name = "tqdm", marker = "python_full_version < '3.14'" }, + { name = "transformers", marker = "python_full_version < '3.14'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2b/6d/677d50855311e4a1286204c5ef2ef5672d01688246a303bde5326311bdad/gliner-0.2.24.tar.gz", hash = "sha256:191a37d1b3d297927c37ae890ce904e9d4e9d3f4c8e19715e77a9876e5f8a575", size = 160568, upload-time = "2025-11-26T18:20:32.867Z" } wheels = [ @@ -2839,7 +2841,7 @@ name = "humanfriendly" version = "10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyreadline3", marker = "sys_platform == 'win32'" }, + { name = "pyreadline3", marker = "python_full_version < '3.14' and sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } wheels = [ @@ -2933,17 +2935,17 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "decorator" }, - { name = "exceptiongroup" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, - { name = "typing-extensions" }, + { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, + { name = "decorator", marker = "python_full_version < '3.11'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "jedi", marker = "python_full_version < '3.11'" }, + { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, + { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, + { name = "pygments", marker = "python_full_version < '3.11'" }, + { name = "stack-data", marker = "python_full_version < '3.11'" }, + { name = "traitlets", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } wheels = [ @@ -2974,18 +2976,18 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "decorator" }, - { name = "ipython-pygments-lexers" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, + { name = "decorator", marker = "python_full_version >= '3.11'" }, + { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, + { name = "jedi", marker = "python_full_version >= '3.11'" }, + { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, + { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, + { name = "psutil", marker = "python_full_version >= '3.11' and sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, + { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "stack-data", marker = "python_full_version >= '3.11'" }, + { name = "traitlets", marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/59/165d3b4d75cc34add3122c4417ecb229085140ac573103c223cd01dde96f/ipython-9.15.0.tar.gz", hash = "sha256:da2819ce2aa83135257df830660b1176d986c3d2876db24df01974fa955b2756", size = 4442580, upload-time = "2026-06-26T11:03:35.913Z" } wheels = [ @@ -2997,7 +2999,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments" }, + { name = "pygments", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -3465,6 +3467,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d3/26/1ef06f56198d631296d646a6223de35bcc6cf9795ceb2442816bc963b84c/langchain_text_splitters-1.1.2-py3-none-any.whl", hash = "sha256:a2de0d799ff31886429fd6e2e0032df275b60ec817c19059a7b46181cc1c2f10", size = 35903, upload-time = "2026-04-16T14:20:38.243Z" }, ] +[[package]] +name = "langcodes" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/75/f9edc5d72945019312f359e69ded9f82392a81d49c5051ed3209b100c0d2/langcodes-3.5.1.tar.gz", hash = "sha256:40bff315e01b01d11c2ae3928dd4f5cbd74dd38f9bd912c12b9a3606c143f731", size = 191084, upload-time = "2025-12-02T16:22:01.627Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/c1/d10b371bcba7abce05e2b33910e39c33cfa496a53f13640b7b8e10bb4d2b/langcodes-3.5.1-py3-none-any.whl", hash = "sha256:b6a9c25c603804e2d169165091d0cdb23934610524a21d226e4f463e8e958a72", size = 183050, upload-time = "2025-12-02T16:21:59.954Z" }, +] + [[package]] name = "langsmith" version = "0.9.8" @@ -3898,15 +3909,15 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "contourpy", version = "1.3.2", source = { registry = "https://pypi.org/simple" } }, - { name = "cycler" }, - { name = "fonttools" }, - { name = "kiwisolver" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "packaging" }, - { name = "pillow" }, - { name = "pyparsing" }, - { name = "python-dateutil" }, + { name = "contourpy", version = "1.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "cycler", marker = "python_full_version < '3.11'" }, + { name = "fonttools", marker = "python_full_version < '3.11'" }, + { name = "kiwisolver", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "packaging", marker = "python_full_version < '3.11'" }, + { name = "pillow", marker = "python_full_version < '3.11'" }, + { name = "pyparsing", marker = "python_full_version < '3.11'" }, + { name = "python-dateutil", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/63/1b/4be5be87d43d327a0cf4de1a56e86f7f84c89312452406cf122efe2839e6/matplotlib-3.10.9.tar.gz", hash = "sha256:fd66508e8c6877d98e586654b608a0456db8d7e8a546eb1e2600efd957302358", size = 34811233, upload-time = "2026-04-24T00:14:13.539Z" } wheels = [ @@ -3990,15 +4001,15 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "contourpy", version = "1.3.3", source = { registry = "https://pypi.org/simple" } }, - { name = "cycler" }, - { name = "fonttools" }, - { name = "kiwisolver" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, - { name = "packaging" }, - { name = "pillow" }, - { name = "pyparsing" }, - { name = "python-dateutil" }, + { name = "contourpy", version = "1.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "cycler", marker = "python_full_version >= '3.11'" }, + { name = "fonttools", marker = "python_full_version >= '3.11'" }, + { name = "kiwisolver", marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "packaging", marker = "python_full_version >= '3.11'" }, + { name = "pillow", marker = "python_full_version >= '3.11'" }, + { name = "pyparsing", marker = "python_full_version >= '3.11'" }, + { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1f/24/080c99d223d158d3a8902769269ab6da5b50f7a0e6e072513907e02b7a6c/matplotlib-3.11.0.tar.gz", hash = "sha256:68c0c7be01b30dcca3638934f7f591df73401235cbdbf0d1ab1c71e7db7f8b57", size = 33251176, upload-time = "2026-06-12T02:29:15.508Z" } wheels = [ @@ -4096,7 +4107,7 @@ name = "miniaudio" version = "1.71" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi" }, + { name = "cffi", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d8/d5/e5439dc08561f73656bfeb3340fc64ab63163e101426593d8fb9a025ff1e/miniaudio-1.71.tar.gz", hash = "sha256:ff51e2887bb673e2e757752b586b3dc924d59aa5fbcae9bbc45f4a111bd3262b", size = 1116480, upload-time = "2026-04-29T21:20:38.182Z" } wheels = [ @@ -4272,7 +4283,7 @@ name = "mlx" version = "0.32.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mlx-metal" }, + { name = "mlx-metal", marker = "sys_platform == 'darwin'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/df/38/a034159df4d21ef7b5721225a2c46092e20a2c627724f57be3e89fa7dbce/mlx-0.32.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:c6feb17e32160b70c7634aab925cf3f8c5c7bebbf99f227c48450478e1008af2", size = 562899, upload-time = "2026-07-07T17:55:25.157Z" }, @@ -4297,18 +4308,18 @@ name = "mlx-audio" version = "0.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub" }, - { name = "miniaudio" }, - { name = "mlx" }, - { name = "mlx-lm" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "sounddevice" }, - { name = "tqdm" }, - { name = "transformers" }, + { name = "huggingface-hub", marker = "sys_platform == 'darwin'" }, + { name = "miniaudio", marker = "sys_platform == 'darwin'" }, + { name = "mlx", marker = "sys_platform == 'darwin'" }, + { name = "mlx-lm", marker = "sys_platform == 'darwin'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and sys_platform == 'darwin'" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*' and sys_platform == 'darwin'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform == 'darwin'" }, + { name = "sounddevice", marker = "sys_platform == 'darwin'" }, + { name = "tqdm", marker = "sys_platform == 'darwin'" }, + { name = "transformers", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/af/1e/f712c9f7997e5051c4da3b658f38162203bb703c750741984c8358c8b897/mlx_audio-0.4.4.tar.gz", hash = "sha256:d751e5f477517e4e7f04de5567318e2fe91b4606af5d7e4b2973603c4777814a", size = 1386491, upload-time = "2026-06-06T15:32:03.504Z" } wheels = [ @@ -4320,14 +4331,14 @@ name = "mlx-lm" version = "0.31.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jinja2" }, - { name = "mlx" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "protobuf" }, - { name = "pyyaml" }, - { name = "sentencepiece" }, - { name = "transformers" }, + { name = "jinja2", marker = "sys_platform == 'darwin'" }, + { name = "mlx", marker = "sys_platform == 'darwin'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and sys_platform == 'darwin'" }, + { name = "protobuf", marker = "sys_platform == 'darwin'" }, + { name = "pyyaml", marker = "sys_platform == 'darwin'" }, + { name = "sentencepiece", marker = "sys_platform == 'darwin'" }, + { name = "transformers", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/84/94/9a38d6b0c6fcca995b9136c94eb7da1e9c5165652edf228b96b29960fa7a/mlx_lm-0.31.3.tar.gz", hash = "sha256:61eb0e3ba09444f77f874aff295401d7ccd20b39495cbbce0c782a15474ce733", size = 304318, upload-time = "2026-04-22T07:37:27.922Z" } wheels = [ @@ -4349,23 +4360,23 @@ name = "mlx-vlm" version = "0.6.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "datasets" }, - { name = "fastapi" }, - { name = "llguidance" }, - { name = "miniaudio" }, - { name = "mlx" }, - { name = "mlx-audio" }, - { name = "mlx-lm" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "opencv-python" }, - { name = "pillow" }, - { name = "python-multipart" }, - { name = "requests" }, - { name = "starlette" }, - { name = "tqdm" }, - { name = "transformers" }, - { name = "uvicorn" }, + { name = "datasets", marker = "sys_platform == 'darwin'" }, + { name = "fastapi", marker = "sys_platform == 'darwin'" }, + { name = "llguidance", marker = "sys_platform == 'darwin'" }, + { name = "miniaudio", marker = "sys_platform == 'darwin'" }, + { name = "mlx", marker = "sys_platform == 'darwin'" }, + { name = "mlx-audio", marker = "sys_platform == 'darwin'" }, + { name = "mlx-lm", marker = "sys_platform == 'darwin'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and sys_platform == 'darwin'" }, + { name = "opencv-python", marker = "sys_platform == 'darwin'" }, + { name = "pillow", marker = "sys_platform == 'darwin'" }, + { name = "python-multipart", marker = "sys_platform == 'darwin'" }, + { name = "requests", marker = "sys_platform == 'darwin'" }, + { name = "starlette", marker = "sys_platform == 'darwin'" }, + { name = "tqdm", marker = "sys_platform == 'darwin'" }, + { name = "transformers", marker = "sys_platform == 'darwin'" }, + { name = "uvicorn", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/de/03810d375be44e04a0889a7709aa72d8f187ec94a5172dea3d91051032e4/mlx_vlm-0.6.4.tar.gz", hash = "sha256:2a911692aedc3861ae26f4057b1c05dcb9abfb954d50123df3ef63eab0c58e29", size = 1453442, upload-time = "2026-07-06T21:11:12.567Z" } wheels = [ @@ -4377,18 +4388,18 @@ name = "mlx-whisper" version = "0.4.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub" }, - { name = "mlx" }, - { name = "more-itertools" }, - { name = "numba" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "tiktoken" }, - { name = "torch", version = "2.12.1", source = { registry = "https://pypi.org/simple" } }, - { name = "tqdm" }, + { name = "huggingface-hub", marker = "sys_platform == 'darwin'" }, + { name = "mlx", marker = "sys_platform == 'darwin'" }, + { name = "more-itertools", marker = "sys_platform == 'darwin'" }, + { name = "numba", marker = "sys_platform == 'darwin'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and sys_platform == 'darwin'" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*' and sys_platform == 'darwin'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform == 'darwin'" }, + { name = "tiktoken", marker = "sys_platform == 'darwin'" }, + { name = "torch", version = "2.12.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "tqdm", marker = "sys_platform == 'darwin'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/22/b7/a35232812a2ccfffcb7614ba96a91338551a660a0e9815cee668bf5743f0/mlx_whisper-0.4.3-py3-none-any.whl", hash = "sha256:6b82b6597a994643a3e5496c7bc229a672e5ca308458455bfe276e76ae024489", size = 890544, upload-time = "2025-08-29T14:56:13.815Z" }, @@ -4778,12 +4789,12 @@ name = "nemotron-ocr" version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, - { name = "pillow" }, - { name = "shapely" }, - { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" } }, - { name = "torchvision", version = "0.26.0", source = { registry = "https://pypi.org/simple" } }, + { name = "huggingface-hub", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "pillow", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "shapely", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "torchvision", version = "0.26.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b7/ef/9dbba22f5de348a5f9c3af0488bf61258872926c40b7d513d71ef465b418/nemotron_ocr-2.0.0.tar.gz", hash = "sha256:84eb64f8af2ae12fbd83e38e482348ecce6a932b30946c873f8b8a95afae7355", size = 155817, upload-time = "2026-05-21T00:06:36.975Z" } wheels = [ @@ -5145,7 +5156,7 @@ resolution-markers = [ "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", ] dependencies = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a3/22/0b4b932655d17a6da1b92fa92ab12844b053bb2ac2475e179ba6f043da1e/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:d20e1734305e9d68889a96e3f35094d733ff1f83932ebe462753973e53a572bf", size = 366066321, upload-time = "2026-02-03T20:44:52.837Z" }, @@ -5163,7 +5174,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, @@ -5175,7 +5186,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -5205,9 +5216,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, + { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -5219,7 +5230,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -5312,9 +5323,9 @@ name = "ocrmac" version = "1.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "pillow" }, - { name = "pyobjc-framework-vision" }, + { name = "click", marker = "sys_platform == 'darwin'" }, + { name = "pillow", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-vision", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5e/07/3e15ab404f75875c5e48c47163300eb90b7409044d8711fc3aaf52503f2e/ocrmac-1.0.1.tar.gz", hash = "sha256:507fe5e4cbd67b2d03f6729a52bbc11f9d0b58241134eb958a5daafd4b9d93d9", size = 1454317, upload-time = "2026-01-08T16:44:26.412Z" } wheels = [ @@ -5361,13 +5372,13 @@ name = "onnxruntime" version = "1.23.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "coloredlogs" }, - { name = "flatbuffers" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, + { name = "coloredlogs", marker = "python_full_version < '3.14'" }, + { name = "flatbuffers", marker = "python_full_version < '3.14'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "sympy" }, + { name = "packaging", marker = "python_full_version < '3.14'" }, + { name = "protobuf", marker = "python_full_version < '3.14'" }, + { name = "sympy", marker = "python_full_version < '3.14'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/35/d6/311b1afea060015b56c742f3531168c1644650767f27ef40062569960587/onnxruntime-1.23.2-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:a7730122afe186a784660f6ec5807138bf9d792fa1df76556b27307ea9ebcbe3", size = 17195934, upload-time = "2025-10-27T23:06:14.143Z" }, @@ -5399,13 +5410,13 @@ name = "onnxruntime-gpu" version = "1.23.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "coloredlogs" }, - { name = "flatbuffers" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "sympy" }, + { name = "coloredlogs", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten')" }, + { name = "flatbuffers", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform != 'darwin'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten'" }, + { name = "packaging", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten')" }, + { name = "protobuf", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten')" }, + { name = "sympy", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/cb/ae/39283748c68a96be4f5f8a9561e0e3ca92af1eae6c2b1c07fb1da5f65cd1/onnxruntime_gpu-1.23.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18de50c6c8eea50acc405ea13d299aec593e46478d7a22cd32cdbbdf7c42899d", size = 300525411, upload-time = "2025-10-22T16:56:08.415Z" }, @@ -5588,10 +5599,10 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "python-dateutil" }, - { name = "pytz" }, - { name = "tzdata" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "python-dateutil", marker = "python_full_version < '3.11'" }, + { name = "pytz", marker = "python_full_version < '3.11'" }, + { name = "tzdata", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -5668,9 +5679,9 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, - { name = "python-dateutil" }, - { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, + { name = "tzdata", marker = "(python_full_version >= '3.11' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } wheels = [ @@ -5792,7 +5803,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -6570,7 +6581,7 @@ name = "pyobjc-framework-cocoa" version = "12.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/51/34/fbe38a204643aa4e1b91391cdce07a34da565a69171ebcad08de7438a556/pyobjc_framework_cocoa-12.2.1.tar.gz", hash = "sha256:b94b37fe5730e5ae1fb0052912cd174e6ec329b0bfba4a012ae5db1014b5864b", size = 3125751, upload-time = "2026-06-19T16:20:05.159Z" } wheels = [ @@ -6590,8 +6601,8 @@ name = "pyobjc-framework-coreml" version = "12.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, - { name = "pyobjc-framework-cocoa" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/1e/7d2db3e4468eb04cc92264be83113d86eea4f96302742437de695a445d6d/pyobjc_framework_coreml-12.2.1.tar.gz", hash = "sha256:ef3c2b6a160891b44173235603d10174929656b9c206d6f2f443fe2aa903c2cb", size = 49272, upload-time = "2026-06-19T16:20:18.459Z" } wheels = [ @@ -6611,8 +6622,8 @@ name = "pyobjc-framework-quartz" version = "12.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, - { name = "pyobjc-framework-cocoa" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3b/f6/2a8b84dbf1fe7c04dd96ea73d991678d4e09a909f51971ecc51629bb2ab4/pyobjc_framework_quartz-12.2.1.tar.gz", hash = "sha256:b3b8b6f71e66147f8ff9e6213864cc8527e3a0b1ee90835b93ce221f4802d9b0", size = 3215521, upload-time = "2026-06-19T16:21:30.199Z" } wheels = [ @@ -6632,10 +6643,10 @@ name = "pyobjc-framework-vision" version = "12.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, - { name = "pyobjc-framework-cocoa" }, - { name = "pyobjc-framework-coreml" }, - { name = "pyobjc-framework-quartz" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-coreml", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/7a/1fdffff1b6bf124b260a2169869f4b71a08b9f6603698f7dec990d5ae5f3/pyobjc_framework_vision-12.2.1.tar.gz", hash = "sha256:debfd59dd7d962a6053bf733370148c11a9ec44091b517a0966f48d81c305879", size = 72683, upload-time = "2026-06-19T16:22:01.102Z" } wheels = [ @@ -7846,14 +7857,14 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "imageio" }, - { name = "lazy-loader" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" } }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "packaging" }, - { name = "pillow" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, - { name = "tifffile", version = "2025.5.10", source = { registry = "https://pypi.org/simple" } }, + { name = "imageio", marker = "python_full_version < '3.11'" }, + { name = "lazy-loader", marker = "python_full_version < '3.11'" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "packaging", marker = "python_full_version < '3.11'" }, + { name = "pillow", marker = "python_full_version < '3.11'" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "tifffile", version = "2025.5.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c7/a8/3c0f256012b93dd2cb6fda9245e9f4bff7dc0486880b248005f15ea2255e/scikit_image-0.25.2.tar.gz", hash = "sha256:e5a37e6cd4d0c018a7a55b9d601357e3382826d3888c10d0213fc63bff977dde", size = 22693594, upload-time = "2025-02-18T18:05:24.538Z" } wheels = [ @@ -7904,15 +7915,15 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "imageio" }, - { name = "lazy-loader" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" } }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, - { name = "packaging" }, - { name = "pillow" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "imageio", marker = "python_full_version >= '3.11'" }, + { name = "lazy-loader", marker = "python_full_version >= '3.11'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "packaging", marker = "python_full_version >= '3.11'" }, + { name = "pillow", marker = "python_full_version >= '3.11'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "tifffile", version = "2026.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "tifffile", version = "2026.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "tifffile", version = "2026.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a1/b4/2528bb43c67d48053a7a649a9666432dc307d66ba02e3a6d5c40f46655df/scikit_image-0.26.0.tar.gz", hash = "sha256:f5f970ab04efad85c24714321fcc91613fcb64ef2a892a13167df2f3e59199fa", size = 22729739, upload-time = "2025-12-20T17:12:21.824Z" } @@ -7976,10 +7987,10 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "joblib" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, - { name = "threadpoolctl" }, + { name = "joblib", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "threadpoolctl", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/c2/a7855e41c9d285dfe86dc50b250978105dce513d6e459ea66a6aeb0e1e0c/scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda", size = 7193136, upload-time = "2025-09-09T08:21:29.075Z" } wheels = [ @@ -8039,12 +8050,12 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "joblib" }, - { name = "narwhals" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "joblib", marker = "python_full_version >= '3.11'" }, + { name = "narwhals", marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "threadpoolctl" }, + { name = "threadpoolctl", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } wheels = [ @@ -8089,7 +8100,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -8151,7 +8162,7 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -8237,7 +8248,7 @@ resolution-markers = [ "(python_full_version == '3.12.*' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [ @@ -8288,8 +8299,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography" }, - { name = "jeepney" }, + { name = "cryptography", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "jeepney", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -8500,7 +8511,7 @@ name = "sounddevice" version = "0.5.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi" }, + { name = "cffi", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2a/f9/2592608737553638fca98e21e54bfec40bf577bb98a61b2770c912aab25e/sounddevice-0.5.5.tar.gz", hash = "sha256:22487b65198cb5bf2208755105b524f78ad173e5ab6b445bdab1c989f6698df3", size = 143191, upload-time = "2026-01-23T18:36:43.529Z" } wheels = [ @@ -8590,8 +8601,8 @@ name = "standard-aifc" version = "3.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "audioop-lts" }, - { name = "standard-chunk" }, + { name = "audioop-lts", marker = "python_full_version >= '3.13'" }, + { name = "standard-chunk", marker = "python_full_version >= '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c4/53/6050dc3dde1671eb3db592c13b55a8005e5040131f7509cef0215212cb84/standard_aifc-3.13.0.tar.gz", hash = "sha256:64e249c7cb4b3daf2fdba4e95721f811bde8bdfc43ad9f936589b7bb2fae2e43", size = 15240, upload-time = "2024-10-30T16:01:31.772Z" } wheels = [ @@ -8612,7 +8623,7 @@ name = "standard-sunau" version = "3.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "audioop-lts" }, + { name = "audioop-lts", marker = "python_full_version >= '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/e3/ce8d38cb2d70e05ffeddc28bb09bad77cfef979eb0a299c9117f7ed4e6a9/standard_sunau-3.13.0.tar.gz", hash = "sha256:b319a1ac95a09a2378a8442f403c66f4fd4b36616d6df6ae82b8e536ee790908", size = 9368, upload-time = "2024-10-30T16:01:41.626Z" } wheels = [ @@ -8624,8 +8635,8 @@ name = "starlette" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "anyio", marker = "sys_platform == 'darwin'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } wheels = [ @@ -8756,7 +8767,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/44/d0/18fed0fc0916578a4463f775b0fbd9c5fed2392152d039df2fb533bfdd5d/tifffile-2025.5.10.tar.gz", hash = "sha256:018335d34283aa3fd8c263bae5c3c2b661ebc45548fde31504016fcae7bf1103", size = 365290, upload-time = "2025-05-10T19:22:34.386Z" } wheels = [ @@ -8774,7 +8785,7 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c5/cb/2f6d79c7576e22c116352a801f4c3c8ace5957e9aced862012430b62e14f/tifffile-2026.3.3.tar.gz", hash = "sha256:d9a1266bed6f2ee1dd0abde2018a38b4f8b2935cb843df381d70ac4eac5458b7", size = 388745, upload-time = "2026-03-03T19:14:38.134Z" } wheels = [ @@ -8801,7 +8812,7 @@ resolution-markers = [ "(python_full_version == '3.12.*' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b7/38/5e2ecef5af2f4fd4a89bb8d6240de9458bab4d51a4cbd97aeb3a0cd618e2/tifffile-2026.6.1.tar.gz", hash = "sha256:626c892c0e899d959b9438e7c0e1491dc154a7fead1f1f37a991724a50eceba9", size = 429694, upload-time = "2026-05-31T23:57:12.165Z" } wheels = [ @@ -9000,20 +9011,20 @@ resolution-markers = [ "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", ] dependencies = [ - { name = "cuda-bindings" }, - { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"] }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" } }, - { name = "nvidia-cudnn-cu13", version = "9.19.0.56", source = { registry = "https://pypi.org/simple" } }, - { name = "nvidia-cusparselt-cu13", version = "0.8.0", source = { registry = "https://pypi.org/simple" } }, - { name = "nvidia-nccl-cu13", version = "2.28.9", source = { registry = "https://pypi.org/simple" } }, - { name = "nvidia-nvshmem-cu13" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "triton", version = "3.6.0", source = { registry = "https://pypi.org/simple" } }, - { name = "typing-extensions" }, + { name = "cuda-bindings", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "filelock", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "fsspec", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "jinja2", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cudnn-cu13", version = "9.19.0.56", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu13", version = "0.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu13", version = "2.28.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu13", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "setuptools", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "sympy", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "triton", version = "3.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f9/1e/18a9b10b4bd34f12d4e561c52b0ae7158707b8193c6cfc0aad2b48167090/torch-2.11.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:1b32ceda909818a03b112006709b02be1877240c31750a8d9c6b7bf5f2d8a6e5", size = 530589207, upload-time = "2026-03-23T18:11:23.756Z" }, @@ -9050,22 +9061,22 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, - { name = "cuda-toolkit", extra = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, + { name = "cuda-bindings", marker = "(python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux')" }, + { name = "cuda-toolkit", extra = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "(python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux')" }, + { name = "filelock", marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "fsspec", marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "jinja2", marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and platform_machine != 'x86_64') or (python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.11' and sys_platform != 'linux')" }, - { name = "nvidia-cublas", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cudnn-cu13", version = "9.20.0.48", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", version = "0.8.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", version = "2.29.7", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "triton", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux'" }, - { name = "typing-extensions" }, + { name = "nvidia-cublas", marker = "(python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cudnn-cu13", version = "9.20.0.48", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusparselt-cu13", version = "0.8.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nccl-cu13", version = "2.29.7", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvshmem-cu13", marker = "(python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux')" }, + { name = "setuptools", marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "sympy", marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "triton", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version != '3.12.*' and sys_platform == 'linux') or (platform_machine != 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/db/ed/ff0c4f8cef63977a646dc80e40c05cae873f4097b12dc87e1cd7e1cecf42/torch-2.12.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:ec56e82be6a8b0c036771a77f7d32ad3c299770571af9815b3dafe61434389d5", size = 87967927, upload-time = "2026-06-17T21:08:43.16Z" }, @@ -9102,9 +9113,9 @@ resolution-markers = [ "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, - { name = "pillow" }, - { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "pillow", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/5e/00/24d8c7845c3f270153fb81395a5135b2778e2538e81d14c6aea5106c689c/torchvision-0.26.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b6f9ad1ecc0eab52647298b379ee9426845f8903703e6127973f8f3d049a798b", size = 7518249, upload-time = "2026-03-23T18:12:51.743Z" }, @@ -9143,8 +9154,8 @@ resolution-markers = [ dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and platform_machine != 'x86_64') or (python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.11' and sys_platform != 'linux')" }, - { name = "pillow" }, - { name = "torch", version = "2.12.1", source = { registry = "https://pypi.org/simple" } }, + { name = "pillow", marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "torch", version = "2.12.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.12.*' or platform_machine != 'x86_64' or sys_platform != 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c9/10/8e3e5a70dded1f86368bc987d93fa0436e73a79060aead75a8783b040ebd/torchvision-0.27.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:68ba63b48af92f06db995adb23d8411993dba1dee705a4e92411b83a00930b7f", size = 1852109, upload-time = "2026-06-17T21:09:34.966Z" }, @@ -9720,9 +9731,9 @@ name = "uvicorn" version = "0.50.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "h11" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "click", marker = "sys_platform == 'darwin'" }, + { name = "h11", marker = "sys_platform == 'darwin'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9f/f6/cc9aadc0e481344a42095d222bfa764122fb8cfba708d1922917bd8bfb01/uvicorn-0.50.2.tar.gz", hash = "sha256:b92bf03509b82bcb9d49e7335b4fd364518ad021c2dc18b4e6a2fec8c955a0bb", size = 93716, upload-time = "2026-07-06T10:38:31.984Z" } wheels = [ From ab72d465e34fe6dd0e5621bb593a985eeb14d2ae Mon Sep 17 00:00:00 2001 From: Nikos Livathinos Date: Tue, 25 Aug 2026 18:58:25 +0200 Subject: [PATCH 03/19] feat: Normalize the input OCR languages to a canonical format which is either the BCP-47 standard or the native input language supported by each OCR engine. Update the tests. Signed-off-by: Nikos Livathinos --- .../.agents/skills/docling/references/cli.md | 2 +- docling/cli/export_utils.py | 6 +- docling/cli/main.py | 44 +- docling/cli/models.py | 17 +- docling/cli/remote.py | 35 +- docling/datamodel/pipeline_options.py | 162 +++- docling/datamodel/service/options.py | 11 +- docling/exceptions.py | 30 + docling/models/base_ocr_model.py | 83 ++ docling/models/stages/ocr/auto_ocr_model.py | 37 + docling/models/stages/ocr/easyocr_model.py | 118 ++- .../models/stages/ocr/kserve_v2_ocr_model.py | 44 +- .../models/stages/ocr/nemotron_ocr_model.py | 53 +- docling/models/stages/ocr/ocr_mac_model.py | 72 +- docling/models/stages/ocr/ppocr_languages.py | 152 ++++ docling/models/stages/ocr/rapid_ocr_model.py | 178 +++-- .../stages/ocr/tesseract_ocr_cli_model.py | 68 +- .../models/stages/ocr/tesseract_ocr_model.py | 65 +- docling/models/stages/ocr/tesseract_utils.py | 147 ++++ docling/utils/model_downloader.py | 5 +- docling/utils/ocr_language.py | 715 +++++++++++++++++ docling/utils/ocr_utils.py | 69 -- tach.toml | 10 + tests/test_backend_webp.py | 4 +- tests/test_cli.py | 2 + tests/test_cli_remote.py | 3 +- tests/test_e2e_nemotron_ocr_conversion.py | 55 +- tests/test_e2e_ocr_conversion.py | 4 +- tests/test_easyocr_lang.py | 65 +- tests/test_kserve_v2_ocr_integration.py | 5 +- tests/test_ocr_engine_language_mapping.py | 270 +++++++ tests/test_ocr_language.py | 755 ++++++++++++++++++ tests/test_ocr_language_behavior.py | 65 ++ ...{test_ocr_utils.py => test_orientation.py} | 0 tests/test_rapid_ocr_lang.py | 43 +- tests/test_tesseract_ocr_lang.py | 93 +++ 36 files changed, 3169 insertions(+), 318 deletions(-) create mode 100644 docling/models/stages/ocr/ppocr_languages.py create mode 100644 docling/models/stages/ocr/tesseract_utils.py create mode 100644 docling/utils/ocr_language.py delete mode 100644 docling/utils/ocr_utils.py create mode 100644 tests/test_ocr_engine_language_mapping.py create mode 100644 tests/test_ocr_language.py create mode 100644 tests/test_ocr_language_behavior.py rename tests/{test_ocr_utils.py => test_orientation.py} (100%) create mode 100644 tests/test_tesseract_ocr_lang.py diff --git a/docling/.agents/skills/docling/references/cli.md b/docling/.agents/skills/docling/references/cli.md index 1a1bf56688..776a759f07 100644 --- a/docling/.agents/skills/docling/references/cli.md +++ b/docling/.agents/skills/docling/references/cli.md @@ -69,7 +69,7 @@ docling scan.pdf --ocr-engine tesserocr --output /tmp/ # needs system Tesserac docling scan.pdf --ocr-engine ocrmac --output /tmp/ # macOS Vision (mac only) docling scan.pdf --force-ocr --output /tmp/ # re-OCR even extractable text docling report.pdf --no-ocr --output /tmp/ # skip OCR (faster) -docling scan.pdf --ocr-lang en --ocr-lang de --output /tmp/ # restrict languages +docling scan.pdf --ocr-lang en,de --output /tmp/ # BCP-47 tags, comma-separated ``` OCR engines are optional dependencies — see diff --git a/docling/cli/export_utils.py b/docling/cli/export_utils.py index 5f00b62716..992111f704 100644 --- a/docling/cli/export_utils.py +++ b/docling/cli/export_utils.py @@ -80,9 +80,13 @@ def _parse_page_range(raw: str | None) -> PageRange | None: def _split_list(raw: str | None) -> list[str] | None: + """Split a comma/semicolon-separated CLI value, dropping blanks. + + Stripping matters: `--ocr-lang "en, de"` must yield `de`, not `" de"`. + """ if raw is None: return None - return re.split(r"[;,]", raw) + return [item.strip() for item in re.split(r"[;,]", raw) if item.strip()] def _is_empty_output(path: Path) -> bool: diff --git a/docling/cli/main.py b/docling/cli/main.py index 5bfc0d77a0..3219f402c0 100644 --- a/docling/cli/main.py +++ b/docling/cli/main.py @@ -8,7 +8,7 @@ from collections.abc import Iterable from enum import Enum from pathlib import Path -from typing import Annotated, Literal, Type, cast +from typing import Annotated, Any, Literal, Type, cast from urllib.parse import urlparse from docling.datamodel.service.responses import ChunkedDocumentResultItem @@ -40,7 +40,7 @@ from docling_core.transforms.visualizer.layout_visualizer import LayoutVisualizer from docling_core.types.doc import ImageRefMode from docling_core.utils.file import resolve_source_to_path -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from rich.console import Console from docling.cli.export_utils import ( @@ -158,6 +158,17 @@ from docling.models.factories.base_factory import BaseFactory from docling.utils.profiling import ProfilingItem + +def _first_error_message(err: ValidationError) -> str: + """The most useful line of a pydantic error, for a typer.BadParameter.""" + errors = err.errors() + if not errors: + return str(err) + message = errors[0].get("msg", "") + # Pydantic prefixes messages raised from a validator with "Value error, ". + return message.removeprefix("Value error, ") or str(err) + + warnings.filterwarnings(action="ignore", category=UserWarning, module="pydantic|torch") warnings.filterwarnings(action="ignore", category=FutureWarning, module="easyocr") @@ -890,7 +901,15 @@ def convert( # noqa: C901 str | None, typer.Option( ..., - help="Provide a comma-separated list of languages used by the OCR engine. Note that each OCR engine has different values for the language names.", + help=( + "Comma-separated list of OCR languages as BCP-47 tags, e.g. " + "'en,de' or 'zh-Hant'. The selected engine's own language codes " + "are accepted too and mean what that engine means by them, so " + "'--ocr-engine rapidocr --ocr-lang ch' is Simplified Chinese. " + "Omit the option to let the engine pick. 'mul' selects a " + "multilingual model on the engines that ship one; to skip OCR " + "entirely use --no-ocr." + ), ), ] = None, psm: Annotated[ @@ -1231,14 +1250,19 @@ def convert( # noqa: C901 resolved_ocr_mode = OcrMode.FULL_PAGE else: resolved_ocr_mode = ocr_mode - ocr_options: OcrOptions = ocr_factory.create_options( # type: ignore - kind=ocr_engine, - mode=resolved_ocr_mode, - ) - + ocr_kwargs: dict[str, Any] = {"mode": resolved_ocr_mode} ocr_lang_list = _split_list(ocr_lang) - if ocr_lang_list is not None: - ocr_options.lang = ocr_lang_list + if ocr_lang_list: + ocr_kwargs["lang"] = ocr_lang_list + try: + ocr_options: OcrOptions = ocr_factory.create_options( # type: ignore + kind=ocr_engine, + **ocr_kwargs, + ) + except ValidationError as err: + raise typer.BadParameter( + _first_error_message(err), param_hint="--ocr-lang" + ) from err if psm is not None and isinstance( ocr_options, TesseractOcrOptions | TesseractCliOcrOptions ): diff --git a/docling/cli/models.py b/docling/cli/models.py index 1e8bc75a0f..907655adc7 100644 --- a/docling/cli/models.py +++ b/docling/cli/models.py @@ -27,7 +27,7 @@ from docling.datamodel.settings import settings from docling.models.stages.ocr.easyocr_model import ( - _resolve_easyocr_recognition_models, + resolve_easyocr_languages, ) from docling.models.stages.ocr.rapid_ocr_model import _parse_rapidocr_model_spec from docling.models.utils.hf_model_download import download_hf_model @@ -119,7 +119,12 @@ def download( typer.Option( ..., "--easyocr-lang", - help="EasyOCR language code to prefetch. Repeat for multiple languages.", + help=( + "OCR language to prefetch for EasyOCR, as a BCP-47 tag " + "(e.g. 'de', 'zh-Hant', 'ru'). EasyOCR's own codes are accepted " + "too and mean what EasyOCR means by them, so 'ch_sim' is " + "Simplified Chinese. Repeat for multiple." + ), ), ] = None, rapidocr_backend_lang: Annotated[ @@ -129,7 +134,11 @@ def download( "--rapidocr-backend-lang", help=( "RapidOCR checkpoint set to prefetch, as ':' " - "(e.g. 'onnxruntime:el', 'torch:korean'). Repeat for multiple. Replaces the default set." + "with a BCP-47 language (e.g. 'onnxruntime:el', 'torch:ko'). " + "PP-OCR's own codes are accepted too, including its script " + "recognizers, which no language tag can name: " + "'onnxruntime:cyrillic', 'torch:ch'. Repeat for multiple. " + "Replaces the default set." ), ), ] = None, @@ -153,7 +162,7 @@ def download( param_hint="--easyocr-lang", ) try: - _resolve_easyocr_recognition_models(easyocr_lang) + resolve_easyocr_languages(easyocr_lang) except ValueError as error: raise typer.BadParameter(str(error), param_hint="--easyocr-lang") from error if rapidocr_backend_lang is not None: diff --git a/docling/cli/remote.py b/docling/cli/remote.py index 7ac3252834..c48bb97d48 100644 --- a/docling/cli/remote.py +++ b/docling/cli/remote.py @@ -24,7 +24,7 @@ ) from docling.cli.main import ChunkerType from docling.datamodel.base_models import InputFormat, OutputFormat -from docling.datamodel.pipeline_options import ProcessingPipeline +from docling.datamodel.pipeline_options import OcrAutoOptions, ProcessingPipeline from docling.datamodel.service.options import ( ConvertDocumentsOptions as ConvertDocumentsRequestOptions, ) @@ -33,9 +33,30 @@ DoclingServiceClient, StatusWatcherKind, ) +from docling.utils.ocr_language import OcrLanguageResolver _log = logging.getLogger(__name__) + +def _canonicalize_ocr_lang(raw: Optional[str]) -> Optional[list[str]]: + """Canonicalize --ocr-lang locally, so a typo fails here and not remotely. + + This command has no `--ocr-engine`, so it cannot pick an engine's own + vocabulary and uses the engine-independent one: native tokens that every + engine agrees on are accepted, and the handful that clash with a BCP-47 tag + of a different language are not. + """ + tags = _split_list(raw) + if not tags: + return None + try: + return OcrLanguageResolver.canonicalize_ocr_language_tags( + tags, kind=OcrAutoOptions.kind + ) + except ValueError as err: + raise typer.BadParameter(str(err), param_hint="--ocr-lang") from err + + _REMOTE_HELP = """\ Convert documents through a remote docling-serve service instead of locally. @@ -188,7 +209,15 @@ def convert_remote( ocr_lang: Annotated[ Optional[str], typer.Option( - help="Comma-separated list of OCR languages (engine-specific names).", + help=( + "Comma-separated list of OCR languages as BCP-47 tags, e.g. " + "'en,de' or 'zh-Hant'. Widely-understood engine names such as " + "'chinese' or 'japan' are accepted too, but engine-specific ones " + "such as 'ch' are not, because this command does not choose the " + "engine. Canonicalized locally before the request is sent, so the " + "remote service must be recent enough to speak BCP-47 OCR " + "languages." + ), ), ] = None, enrich_code: Annotated[ @@ -306,7 +335,7 @@ def convert_remote( "to_formats": to_formats, "do_ocr": ocr, "force_ocr": force_ocr, - "ocr_lang": _split_list(ocr_lang), + "ocr_lang": _canonicalize_ocr_lang(ocr_lang), "do_table_structure": tables, "pipeline": pipeline, "do_code_enrichment": enrich_code, diff --git a/docling/datamodel/pipeline_options.py b/docling/datamodel/pipeline_options.py index 9b0bfb6084..7d6abb1bcd 100644 --- a/docling/datamodel/pipeline_options.py +++ b/docling/datamodel/pipeline_options.py @@ -72,6 +72,7 @@ ObjectDetectionEngineOptionsMixin, ) from docling.models.inference_engines.vlm.base import VlmEngineOptionsMixin +from docling.utils.ocr_language import OcrLanguageResolver _log = logging.getLogger(__name__) @@ -211,8 +212,18 @@ class OcrOptions(BaseOptions): lang: Annotated[ list[str], Field( - description="List of OCR languages to use. The format must match the values of the OCR engine of choice.", - examples=[["deu", "eng"]], + description=( + "OCR languages as BCP-47 tags (e.g. `en`, `de-DE`, `zh-Hant`), in " + "order of preference. Tags are canonicalized to a language-script " + "pair, so `deu`, `ger`, `de` and `de-DE` are all `de-Latn`. An empty " + "list means the engine's own default, which for Tesseract is " + "per-page script detection. One tag carries engine-independent " + "meaning and must be used alone: `mul`, the engine's broadest " + "multilingual model, on the engines that ship one. " + "A language the selected engine has no model for raises an error " + "rather than falling back silently." + ), + examples=[["de", "en"], ["zh-Hans"], []], ), ] @@ -244,6 +255,24 @@ class OcrOptions(BaseOptions): ), ] = False + model_config = ConfigDict( + validate_assignment=True, + validate_default=True, + ) + + @field_validator("lang", mode="after") + @classmethod + def _canonicalize_lang(cls, value: list[str]) -> list[str]: + """Rewrite every entry into its canonical BCP-47 form. + + Declared once on the base: pydantic collects field validators by field + name across the MRO, so it still fires for the subclasses that redefine + `lang` with their own default. `cls` is the concrete options class, so + `cls.kind` names the engine -- this is the only point that knows both the + engine and the string the user wrote. + """ + return OcrLanguageResolver.canonicalize_ocr_language_tags(value, kind=cls.kind) + @model_validator(mode="after") def _apply_force_full_page_ocr(self) -> "OcrOptions": r""" @@ -253,7 +282,9 @@ def _apply_force_full_page_ocr(self) -> "OcrOptions": with warnings.catch_warnings(): # deprecated force_full_page_ocr warnings.filterwarnings("ignore", category=DeprecationWarning) forced = self.force_full_page_ocr - if forced: + # `validate_assignment` re-runs this validator on every assignment, so + # the write must be skipped once `mode` already holds the forced value. + if forced and self.mode is not OcrMode.FULL_PAGE: self.mode = OcrMode.FULL_PAGE return self @@ -263,13 +294,16 @@ class OcrAutoOptions(OcrOptions): When this option is used, Docling probes the runtime environment at pipeline initialization and selects the best available OCR engine - (e.g., EasyOCR if GPU is present, Tesseract otherwise). Language - settings are deferred to the chosen engine's defaults. + (e.g., EasyOCR if GPU is present, Tesseract otherwise). The requested + languages are forwarded to whichever engine is chosen, and an engine with + no model for them is skipped in favour of the next candidate. Notes: - The `lang` field is intentionally defaulted to an empty list. - To control language selection, specify an explicit OCR engine - option class instead. + `lang` is forwarded to whichever engine is selected. The default empty + list means "each engine's own default model", so leaving it alone + reproduces the behaviour of picking that engine by hand. An engine that + cannot serve the requested language is treated as unavailable and the + next candidate is probed. """ kind: ClassVar[Literal["auto"]] = "auto" @@ -277,9 +311,12 @@ class OcrAutoOptions(OcrOptions): list[str], Field( description=( - "The automatic OCR engine will use the default values of the engine. Please specify the engine " - "explicitly to change the language selection." - ) + "OCR languages as BCP-47 tags, forwarded to the automatically " + "selected engine. The default empty list leaves the choice of model " + "to that engine. Engines with no model for the requested language " + "are skipped during selection." + ), + examples=[[], ["de", "en"]], ), ] = [] @@ -297,18 +334,21 @@ class RapidOcrOptions(OcrOptions): list[str], Field( description=( - "Recognition language. RapidOCR uses a single language per run; if more than one " - "value is given only the first is used. Accepted values resolve to a PP-OCR " - "recognizer: PP-OCRv6 covers ~52 language codes (e.g. 'ch', 'en', 'de', 'fr', " - "'japan'; the docling defaults 'chinese'/'english' map to 'ch'/'en'). Script-family " - "names route to PP-OCRv5 on the onnxruntime/openvino/paddle backends ('arabic', " - "'ch', 'cyrillic', 'devanagari', 'el', 'en', 'eslav', 'korean', 'latin', 'ta', " - "'te', 'th') or to PP-OCRv4 on the torch backend ('arabic', 'cyrillic', " - "'devanagari', 'ka', 'korean', 'latin', 'ta', 'te'). A language the resolved " - "backend cannot serve raises an error rather than falling back silently." - ) + "Recognition language as a BCP-47 tag. RapidOCR runs a single " + "language per run; if more than one tag is given the first is used " + "and the rest are ignored with a warning. Tags are mapped onto a " + "PP-OCR recognizer: PP-OCRv6 covers ~52 languages, and a language " + "PP-OCR serves only through a script-wide recognizer routes to " + "PP-OCRv5 (onnxruntime/openvino/paddle) or PP-OCRv4 (torch). " + "PP-OCR's script recognizers can also be named directly with their " + "own tokens (`latin`, `cyrillic`, `arabic`, `devanagari`). An empty " + "list selects the Simplified Chinese default; `mul` is not " + "supported. A language the resolved backend cannot serve raises an " + "error rather than falling back silently." + ), + examples=[["zh-Hans"], ["en"], ["cyrillic"]], ), - ] = ["chinese"] + ] = ["zh-Hans"] backend: Annotated[ Literal["onnxruntime", "openvino", "paddle", "torch"], Field( @@ -416,8 +456,14 @@ class NemotronOcrOptions(OcrOptions): list[str], Field( description=( - "List of OCR languages. nemotron-OCR-v2 supports 'english' and 'multilingual'" - ) + "Recognition language as a BCP-47 tag. nemotron-OCR-v2 ships two " + "recognizers: `en` and an empty list both select the English " + "model, while `mul` and the languages the multilingual model " + "covers (`zh-Hans`, `zh-Hant`, `ja`, `ko`, `ru`) select the " + "multilingual one. Any other language raises; use `mul` to opt " + "into the multilingual model explicitly." + ), + examples=[["en"], ["mul"]], ), ] = [] merge_level: Annotated[ @@ -451,11 +497,16 @@ class EasyOcrOptions(OcrOptions): list[str], Field( description=( - "List of language codes for OCR. EasyOCR supports 80+ languages. Use ISO 639-1 codes " - "(e.g., `en`, `fr`, `de`). Multiple languages can be specified for multilingual documents." - ) + "OCR languages as BCP-47 tags. EasyOCR covers 80+ languages and " + "runs several at once, but they must share a recognition model, so " + "keep the list short and script-consistent. Each language is routed " + "to the recognition network of its script, so `ru` reaches the " + "Cyrillic model. `mul` is not supported -- list the languages " + "explicitly." + ), + examples=[["fr", "de", "es", "en"], ["ru", "uk"]], ), - ] = ["fr", "de", "es", "en"] + ] = ["fr-Latn", "de-Latn", "es-Latn", "en-Latn"] use_gpu: Annotated[ bool | None, Field( @@ -524,11 +575,18 @@ class TesseractCliOcrOptions(OcrOptions): list[str], Field( description=( - "List of Tesseract language codes. Use 3-letter ISO 639-2 codes (e.g., `eng`, `fra`, `deu`). " - "Multiple languages enable multilingual OCR. Requires corresponding Tesseract language data files." - ) + "OCR languages as BCP-47 tags, mapped onto the installed tessdata " + "files (`de` becomes `deu`, `zh-Hant` becomes `chi_tra`). Multiple " + "languages enable multilingual OCR and are joined in the order " + "given, which Tesseract treats as preference order. A `script/` " + "traineddata file can be named directly (`script/Cyrillic`), and an " + "empty list runs orientation and script detection per page " + "(requires the `osd` traineddata). Languages without an installed " + "traineddata file raise at construction time." + ), + examples=[["fr", "de", "es", "en"], []], ), - ] = ["fra", "deu", "spa", "eng"] + ] = ["fr-Latn", "de-Latn", "es-Latn", "en-Latn"] tesseract_cmd: Annotated[ str, Field( @@ -569,11 +627,18 @@ class TesseractOcrOptions(OcrOptions): list[str], Field( description=( - "List of Tesseract language codes. Use 3-letter ISO 639-2 codes (e.g., `eng`, `fra`, `deu`). " - "Multiple languages enable multilingual OCR. Requires corresponding Tesseract language data files." - ) + "OCR languages as BCP-47 tags, mapped onto the installed tessdata " + "files (`de` becomes `deu`, `zh-Hant` becomes `chi_tra`). Multiple " + "languages enable multilingual OCR and are joined in the order " + "given, which Tesseract treats as preference order. A `script/` " + "traineddata file can be named directly (`script/Cyrillic`), and an " + "empty list runs orientation and script detection per page " + "(requires the `osd` traineddata). Languages without an installed " + "traineddata file raise at construction time." + ), + examples=[["fr", "de", "es", "en"], []], ), - ] = ["fra", "deu", "spa", "eng"] + ] = ["fr-Latn", "de-Latn", "es-Latn", "en-Latn"] path: Annotated[ str | None, Field( @@ -605,11 +670,14 @@ class OcrMacOptions(OcrOptions): list[str], Field( description=( - "List of language locale codes for macOS OCR. Use format `language-REGION` (e.g., `en-US`, `fr-FR`). " - "Leverages native macOS Vision framework for OCR on Apple platforms." - ) + "OCR languages as BCP-47 tags, matched against the recognition " + "languages the running macOS reports (`de` becomes `de-DE`, `pt` " + "becomes `pt-BR`). An empty list hands the choice to Vision's own " + "automatic behaviour. `mul` is not supported." + ), + examples=[["fr", "de", "es", "en"], []], ), - ] = ["fr-FR", "de-DE", "es-ES", "en-US"] + ] = ["fr-Latn", "de-Latn", "es-Latn", "en-Latn"] recognition: Annotated[ str, Field( @@ -660,11 +728,17 @@ class KserveV2OcrOptions(OcrOptions, KserveV2OptionsMixin): list[str], Field( description=( - "List of OCR languages. Note: Language selection depends on the deployed model. " - "This parameter is passed to the server but may not be used by all models." - ) + "Recognition language as a BCP-47 tag, mapped to the PP-OCR token " + "the server is expected to understand. A single language is sent; " + "if more than one tag is given the first is used and the rest are " + "ignored with a warning. An empty list sends the Simplified " + "Chinese default. Coverage is checked against the PP-OCR token universe, " + "which is only a best-effort proxy for what the deployed model " + "actually serves." + ), + examples=[["en"], ["zh-Hans"]], ), - ] = ["english", "chinese"] + ] = ["en-Latn"] scale: Annotated[ float, diff --git a/docling/datamodel/service/options.py b/docling/datamodel/service/options.py index 786c852459..2fda4ed83e 100644 --- a/docling/datamodel/service/options.py +++ b/docling/datamodel/service/options.py @@ -375,12 +375,13 @@ class ConvertDocumentsOptions(BaseModel): Optional[list[str]], Field( description=( - "List of languages used by the OCR engine. " - "Note that each OCR engine has " - "different values for the language names. String or list of strings. " - "Optional, defaults to empty." + "OCR languages as BCP-47 tags (e.g. `en`, `de-DE`, `zh-Hant`), in " + "order of preference. The service canonicalizes them to a " + "language-script pair, so `deu`, `ger` and `de-DE` are all German. " + "The reserved tag `mul` must be used alone. Optional; " + "the selected engine's default applies when omitted or empty." ), - examples=[["fr", "de", "es", "en"]], + examples=[["fr", "de", "es", "en"], ["zh-Hant"], []], ), ] = None diff --git a/docling/exceptions.py b/docling/exceptions.py index 1d4b576c33..4700158d81 100644 --- a/docling/exceptions.py +++ b/docling/exceptions.py @@ -1,3 +1,6 @@ +from collections.abc import Sequence + + class BaseError(RuntimeError): pass @@ -25,3 +28,30 @@ class SecurityError(BaseError): class AcceleratorDeviceNotAvailableError(BaseError): """Raised when an explicitly requested accelerator device is not available.""" + + +class OcrLanguageNotSupportedError(BaseError): + """Raised when an OCR engine has no model for a requested language. + + Docling never silently substitutes a different recognizer: when the + canonicalized request cannot be served, the engine says so and names what it + does support. + """ + + def __init__( + self, + engine: str, + language: str, + supported: "Sequence[str] | None" = None, + detail: str | None = None, + ): + self.engine = engine + self.language = language + self.detail = detail + self.supported = list(supported) if supported is not None else [] + message = f"{engine} has no model for the OCR language {language!r}." + if detail: + message = f"{message} {detail}" + if self.supported: + message = f"{message} Supported: {', '.join(self.supported)}." + super().__init__(message) diff --git a/docling/models/base_ocr_model.py b/docling/models/base_ocr_model.py index 7b75411b60..2c3df98702 100644 --- a/docling/models/base_ocr_model.py +++ b/docling/models/base_ocr_model.py @@ -4,6 +4,7 @@ from collections.abc import Iterable from enum import Enum from pathlib import Path +from typing import ClassVar import numpy as np from docling_core.types.doc import BoundingBox, CoordOrigin, Size @@ -17,7 +18,14 @@ from docling.datamodel.document import ConversionResult from docling.datamodel.pipeline_options import OcrMode, OcrOptions from docling.datamodel.settings import settings +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_model import BaseModelWithOptions, BasePageModel +from docling.utils.ocr_language import ( + MULTIPLE, + OcrLanguage, + OcrLanguageResolver, + OcrLanguageSupport, +) _log = logging.getLogger(__name__) @@ -45,6 +53,10 @@ class BaseOcrModel(BasePageModel, BaseModelWithOptions): DEFAULT_DILATION_SIZE = 20 + #: What this engine can do with a language request. Engines override it; + #: the conservative default suits a single-model, single-language engine. + language_support: ClassVar[OcrLanguageSupport] = OcrLanguageSupport() + def __init__( self, *, @@ -56,6 +68,77 @@ def __init__( self.enabled = enabled self.options = options + # `options.lang` is already canonical + self.languages: tuple[OcrLanguage, ...] = ( + OcrLanguageResolver.parse_ocr_languages( + options.lang, kind=type(options).kind + ) + ) + + @property + def _engine_name(self) -> str: + """Human-readable engine name for coverage errors.""" + return type(self).__name__.removesuffix("Model") + + def supported_ocr_languages(self) -> list[str]: + """Canonical tags this *instance* can serve, for error messages. + + May be runtime-derived: the installed tessdata files, the selected + RapidOCR backend, the macOS version. An empty list means "unknown". + """ + return [] + + def map_ocr_language(self, language: OcrLanguage) -> str | list[str]: + """Map one canonical tag onto this engine's native code(s). + + A list covers an engine that answers one request with several native + codes; most engines return a single code. + + Raises: + OcrLanguageNotSupportedError: The engine has no model for it. + """ + if language.is_passthrough or language.language == MULTIPLE: + # A passthrough names a script recognizer of *some* engine; an engine + # that has not overridden this method does not have one. + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + detail="This engine needs an explicit language.", + ) + # Most ISO-639 engines want the primary subtag and nothing else. + return language.language + + def resolve_ocr_languages(self) -> list[str]: + """Turn the canonical request into the native codes to hand the engine. + + An empty request stays empty: `lang=[]` means "the engine's own default", + and each engine decides what that is when it reads the result. + + Applies the two uniform policies: too many languages for the engine are + dropped with a warning (list order is preference order), and a language + with no model is an error, never a silent substitution. + """ + languages = list(self.languages) + if not self.language_support.multiple_languages and len(languages) > 1: + _log.warning( + "%s handles one OCR language at a time. Using %s and ignoring %s; " + "the order of `lang` is the order of preference.", + self._engine_name, + [languages[0].tag], + [lang.tag for lang in languages[1:]], + ) + languages = languages[:1] + + native: list[str] = [] + for language in languages: + mapped = self.map_ocr_language(language) + codes = [mapped] if isinstance(mapped, str) else list(mapped) + for code in codes: + if code not in native: + native.append(code) + return native + def get_ocr_rects(self, page: Page) -> list[BoundingBox]: r""" Produce the input rects for the OCR according to the logic for each OcrMode diff --git a/docling/models/stages/ocr/auto_ocr_model.py b/docling/models/stages/ocr/auto_ocr_model.py index b12be6460a..20c7ba2675 100644 --- a/docling/models/stages/ocr/auto_ocr_model.py +++ b/docling/models/stages/ocr/auto_ocr_model.py @@ -15,6 +15,7 @@ OcrOptions, RapidOcrOptions, ) +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_ocr_model import BaseOcrModel from docling.models.stages.ocr.easyocr_model import EasyOcrModel from docling.models.stages.ocr.nemotron_ocr_model import NemotronOcrModel @@ -41,6 +42,8 @@ def __init__( self.options: OcrAutoOptions self._engine: Optional[BaseOcrModel] = None + # Why each candidate was passed over, for the aggregated error below. + rejected: list[tuple[str, str]] = [] if self.enabled: if "darwin" == sys.platform: try: @@ -51,12 +54,17 @@ def __init__( artifacts_path=artifacts_path, options=OcrMacOptions( mode=self.options.mode, + lang=self.options.lang, ), accelerator_options=accelerator_options, ) _log.info("Auto OCR model selected ocrmac.") except ImportError: _log.info("ocrmac cannot be used because ocrmac is not installed.") + rejected.append(("ocrmac", "not installed")) + except OcrLanguageNotSupportedError as exc: + _log.info("Auto OCR: skipping ocrmac: %s", exc) + rejected.append(("ocrmac", str(exc))) if "linux" == sys.platform: try: @@ -69,14 +77,19 @@ def __init__( artifacts_path=artifacts_path, options=NemotronOcrOptions( mode=self.options.mode, + lang=self.options.lang, ), accelerator_options=accelerator_options, ) _log.info("Auto OCR model selected nemotron.") except ImportError: _log.info("Nemotron cannot be used because it is not installed.") + rejected.append(("nemotron", "not installed")) except (RuntimeError, FileNotFoundError) as exc: + # OcrLanguageNotSupportedError is a BaseError, hence a + # RuntimeError, so this arm covers coverage failures too. _log.warning("Nemotron OCR cannot be used: %s", exc) + rejected.append(("nemotron", str(exc))) if self._engine is None: try: @@ -89,6 +102,7 @@ def __init__( options=RapidOcrOptions( backend="onnxruntime", mode=self.options.mode, + lang=self.options.lang, ), accelerator_options=accelerator_options, ) @@ -97,6 +111,10 @@ def __init__( _log.info( "rapidocr cannot be used because onnxruntime is not installed." ) + rejected.append(("rapidocr (onnxruntime)", "not installed")) + except OcrLanguageNotSupportedError as exc: + _log.info("Auto OCR: skipping rapidocr (onnxruntime): %s", exc) + rejected.append(("rapidocr (onnxruntime)", str(exc))) if self._engine is None: try: @@ -107,12 +125,17 @@ def __init__( artifacts_path=artifacts_path, options=EasyOcrOptions( mode=self.options.mode, + lang=self.options.lang, ), accelerator_options=accelerator_options, ) _log.info("Auto OCR model selected easyocr.") except ImportError: _log.info("easyocr cannot be used because it is not installed.") + rejected.append(("easyocr", "not installed")) + except OcrLanguageNotSupportedError as exc: + _log.info("Auto OCR: skipping easyocr: %s", exc) + rejected.append(("easyocr", str(exc))) if self._engine is None: try: @@ -125,6 +148,7 @@ def __init__( options=RapidOcrOptions( backend="torch", mode=self.options.mode, + lang=self.options.lang, ), accelerator_options=accelerator_options, ) @@ -133,8 +157,21 @@ def __init__( _log.info( "rapidocr cannot be used because rapidocr or torch is not installed." ) + rejected.append(("rapidocr (torch)", "not installed")) + except OcrLanguageNotSupportedError as exc: + _log.info("Auto OCR: skipping rapidocr (torch): %s", exc) + rejected.append(("rapidocr (torch)", str(exc))) if self._engine is None: + if any(reason != "not installed" for _, reason in rejected): + listed = "\n".join( + f" - {name}: {reason}" for name, reason in rejected + ) + raise OcrLanguageNotSupportedError( + "Automatic OCR engine selection", + ", ".join(self.options.lang), + detail=f"No installed engine can serve it:\n{listed}", + ) _log.warning("No OCR engine found. Please review the install details.") def __call__( diff --git a/docling/models/stages/ocr/easyocr_model.py b/docling/models/stages/ocr/easyocr_model.py index 4fa6c06915..9e2fa5571a 100644 --- a/docling/models/stages/ocr/easyocr_model.py +++ b/docling/models/stages/ocr/easyocr_model.py @@ -18,16 +18,48 @@ OcrOptions, ) from docling.datamodel.settings import settings -from docling.exceptions import SecurityError +from docling.exceptions import OcrLanguageNotSupportedError, SecurityError from docling.models.base_ocr_model import BaseOcrModel from docling.utils.accelerator_utils import decide_device +from docling.utils.ocr_language import ( + OcrLanguage, + OcrLanguageResolver, + OcrLanguageSupport, +) from docling.utils.profiling import TimeRecorder from docling.utils.utils import download_url_with_progress _log = logging.getLogger(__name__) -def _resolve_easyocr_recognition_models(languages: Iterable[str]) -> List[str]: +# Canonical tag -> EasyOCR code, where EasyOCR deviates from ISO 639-1. +_EASYOCR_LANGUAGE_CODES: dict[str, str] = { + "zh-Hans": "ch_sim", + "zh-Hant": "ch_tra", + "sr-Cyrl": "rs_cyrillic", + "sr-Latn": "rs_latin", + "tg-Cyrl": "tjk", + "fil-Latn": "tl", + # EasyOCR's `ang` is Angika and its `mah` is Magahi, both Devanagari. CLDR + # gives Angika a likely script of Latin and normalizes `mah` to Marshallese, + # so neither is reachable without an explicit entry. + "anp-Deva": "ang", + "mag-Deva": "mah", + # Tabasaran is written in Cyrillic; CLDR's likely script for it is Latin. + "tab-Cyrl": "tab", +} + +_EASYOCR_CODE_TO_TAG: dict[str, str] = { + code: tag for tag, code in _EASYOCR_LANGUAGE_CODES.items() +} + + +def _easyocr_language_models() -> dict[str, str]: + """EasyOCR language code -> the recognition checkpoint that serves it. + + Doubles as EasyOCR's supported-language vocabulary: a code absent from this + mapping has no recognizer. + """ from easyocr.config import ( arabic_lang_list, bengali_lang_list, @@ -58,6 +90,43 @@ def _resolve_easyocr_recognition_models(languages: Iterable[str]) -> List[str]: "kn": "kannada_g2", } ) + return language_models + + +def _easyocr_code(language: OcrLanguage) -> Optional[str]: + """The EasyOCR code for a canonical tag, or `None` when there is no model.""" + code = _EASYOCR_LANGUAGE_CODES.get(language.tag) + if code is None: + # EasyOCR's codes are language-based, so the primary subtag only + # identifies the right model when the script is the usual one: its `az` + # is Latin Azerbaijani, not `az-Cyrl`. + if not language.has_default_script: + return None + code = language.language + return code if code in _easyocr_language_models() else None + + +def resolve_easyocr_languages(tags: Iterable[str]) -> List[str]: + """Canonicalize language tags into the EasyOCR codes they name. + + Accepts EasyOCR's own codes as well as BCP-47, matching what + `EasyOcrOptions.lang` accepts. Used by the prefetcher, which has no model + instance to ask. + """ + codes: List[str] = [] + for tag in tags: + language = OcrLanguageResolver.parse_ocr_language(tag, EasyOcrOptions.kind) + code = _easyocr_code(language) + if code is None: + raise ValueError(f"Unsupported EasyOCR language: {tag}") + if code not in codes: + codes.append(code) + return codes + + +def _resolve_easyocr_recognition_models(languages: Iterable[str]) -> List[str]: + """Map EasyOCR language codes onto the checkpoints the prefetcher must fetch.""" + language_models = _easyocr_language_models() model_names: List[str] = [] for language in languages: @@ -73,6 +142,8 @@ def _resolve_easyocr_recognition_models(languages: Iterable[str]) -> List[str]: class EasyOcrModel(BaseOcrModel): _model_repo_folder = "EasyOcr" + language_support = OcrLanguageSupport(multiple_languages=True) + def __init__( self, enabled: bool, @@ -90,6 +161,7 @@ def __init__( # multiplier for 72 dpi; the default 3.0 == 216 dpi. self.scale = self.options.scale + self._native_langs: List[str] = [] if self.enabled: try: @@ -100,6 +172,8 @@ def __init__( "Alternatively, Docling has support for other OCR engines. See the documentation." ) + self._native_langs = self.resolve_ocr_languages() + if self.options.use_gpu is None: device = decide_device(accelerator_options.device) # Enable easyocr GPU if running on CUDA, MPS @@ -128,7 +202,7 @@ def __init__( if self.options.suppress_mps_warnings: warnings.filterwarnings("ignore", message=".*pin_memory.*MPS.*") self.reader = easyocr.Reader( - lang_list=self.options.lang, + lang_list=self._native_langs, gpu=use_gpu, model_storage_directory=model_storage_directory, recog_network=self.options.recog_network, @@ -136,6 +210,34 @@ def __init__( verbose=False, ) + def supported_ocr_languages(self) -> List[str]: + tags = set() + for code in _easyocr_language_models(): + tag = _easyocr_code_to_tag(code) + if tag is not None: + tags.add(tag) + return sorted(tags) + + def map_ocr_language(self, language: OcrLanguage) -> str | List[str]: + if language.is_passthrough or language.is_multilingual: + # EasyOCR's codes are all language codes: it has neither a script + # recognizer nor a multilingual model. + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + detail="EasyOCR has no multilingual model; list the languages explicitly.", + ) + + code = _easyocr_code(language) + if code is None: + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + ) + return code + @staticmethod def download_models( detection_models: List[str] = ["craft"], @@ -255,3 +357,13 @@ def __call__( @classmethod def get_options_type(cls) -> Type[OcrOptions]: return EasyOcrOptions + + +def _easyocr_code_to_tag(code: str) -> Optional[str]: + """Render one EasyOCR language code back as a canonical tag.""" + if code in _EASYOCR_CODE_TO_TAG: + return _EASYOCR_CODE_TO_TAG[code] + try: + return OcrLanguageResolver.parse_ocr_language(code, EasyOcrOptions.kind).tag + except ValueError: + return None diff --git a/docling/models/stages/ocr/kserve_v2_ocr_model.py b/docling/models/stages/ocr/kserve_v2_ocr_model.py index 859c142e08..720ae39d98 100644 --- a/docling/models/stages/ocr/kserve_v2_ocr_model.py +++ b/docling/models/stages/ocr/kserve_v2_ocr_model.py @@ -21,12 +21,26 @@ from docling.datamodel.kserve_transport_utils import resolve_kserve_transport_base_url from docling.datamodel.pipeline_options import KserveV2OcrOptions, OcrOptions from docling.datamodel.settings import settings +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_ocr_model import BaseOcrModel from docling.models.inference_engines.common import KserveV2Client, KserveV2HttpClient +from docling.models.stages.ocr.ppocr_languages import ( + PPOCR_DEFAULT_TOKEN, + PPOCRV4_LANGS, + PPOCRV5_LANGS, + PPOCRV6_LANGS, + ppocr_supported_tags, + ppocr_token, +) +from docling.utils.ocr_language import OcrLanguage, OcrLanguageSupport from docling.utils.profiling import TimeRecorder _log = logging.getLogger(__name__) +# The client cannot know what the deployed model serves, so coverage is checked +# against the whole PP-OCR token universe: a best-effort guard against typos. +_KSERVE_PPOCR_VOCABULARY = PPOCRV4_LANGS | PPOCRV5_LANGS | PPOCRV6_LANGS + class KserveV2OcrModel(BaseOcrModel): """OCR model using KServe v2 API (Triton Inference Server, KServe, etc.). @@ -43,13 +57,14 @@ class KserveV2OcrModel(BaseOcrModel): _kserve_client: Client for communicating with the KServe v2 endpoint. """ + language_support = OcrLanguageSupport(multiple_languages=False) + def __init__( self, enabled: bool, artifacts_path: Optional[Path], options: KserveV2OcrOptions, accelerator_options: AcceleratorOptions, - default_language: str = "en", ): """Initialize the KServe v2 OCR model. @@ -72,9 +87,34 @@ def __init__( self._initialize_client() # Prepare the lang_input during the initialization as it stays the same for all requests - self._lang = options.lang[0] if len(options.lang) > 0 else default_language + self._lang = self.resolve_ocr_languages()[0] self._lang_input = np.array([[self._lang]], dtype=object) + def supported_ocr_languages(self) -> List[str]: + return ppocr_supported_tags(_KSERVE_PPOCR_VOCABULARY, KserveV2OcrOptions.kind) + + def resolve_ocr_languages(self) -> List[str]: + # An empty `lang` list means "the engine's own default", which for PP-OCR + # is the Simplified Chinese recognizer. + if not self.languages: + return [PPOCR_DEFAULT_TOKEN] + return super().resolve_ocr_languages() + + def map_ocr_language(self, language: OcrLanguage) -> str: + token = ppocr_token(language, _KSERVE_PPOCR_VOCABULARY) + if token is None: + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + detail=( + "No PP-OCR recognizer covers it; name the languages explicitly." + ) + if language.is_multilingual + else None, + ) + return token + def _initialize_client(self) -> None: """Initialize the KServe v2 client for remote inference.""" base_url = resolve_kserve_transport_base_url( diff --git a/docling/models/stages/ocr/nemotron_ocr_model.py b/docling/models/stages/ocr/nemotron_ocr_model.py index cb094a108a..1012ced12c 100644 --- a/docling/models/stages/ocr/nemotron_ocr_model.py +++ b/docling/models/stages/ocr/nemotron_ocr_model.py @@ -20,9 +20,11 @@ OcrOptions, ) from docling.datamodel.settings import settings +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_ocr_model import BaseOcrModel from docling.models.utils.hf_model_download import download_hf_model from docling.utils.accelerator_utils import decide_device +from docling.utils.ocr_language import OcrLanguage, OcrLanguageSupport from docling.utils.profiling import TimeIntervalRecorder _log = logging.getLogger(__name__) @@ -33,7 +35,11 @@ _NEMOTRON_OCR_ENGLISH = "english" _NEMOTRON_OCR_MULTILINGUAL = "multilingual" -_NEMOTRON_OCR_ENGLISH_GROUP = ["en", "eng", "english"] + +# Canonical tags the multilingual recognizer is trained on. +_NEMOTRON_OCR_MULTILINGUAL_TAGS = frozenset( + {"zh-Hans", "zh-Hant", "ja-Jpan", "ko-Kore", "ru-Cyrl"} +) # Mappings of nemotron language to the artifacts subdir _NEMOTRON_OCR_LANG_TO_ARTIFACT_PATHS = { @@ -46,25 +52,6 @@ def nemotron_ocr_model_dir() -> str: return _NEMOTRON_OCR_REPO_ID.replace("/", "--") -def resolve_nemotronocr_language(req_languages: list[str] | None) -> str: - r""" - Map requested languages onto the nemotron-ocr language info - """ - if not req_languages: - # Use english by default - return _NEMOTRON_OCR_ENGLISH - - # Map request language to nemotron language - for language in req_languages: - # "en-US" / "en_US" -> "en" - normalized = language.strip().lower().replace("_", "-").split("-")[0] - - # Use the multilingual model to cover english and any non-english language - if normalized not in _NEMOTRON_OCR_ENGLISH_GROUP: - return _NEMOTRON_OCR_MULTILINGUAL - return _NEMOTRON_OCR_ENGLISH - - class NemotronOcrPrediction(TypedDict): """Exact prediction schema returned by `nemotron_ocr`.""" @@ -106,6 +93,8 @@ class _BufferedRect: class NemotronOcrModel(BaseOcrModel): r"""Wrapper for Nvidia's nemotron-ocr-v2 model""" + language_support = OcrLanguageSupport(multiple_languages=False) + def __init__( self, enabled: bool, @@ -138,8 +127,10 @@ def __init__( "Python 3.12 and CUDA 13.x." ) from exc - # Resolve the request language - language = resolve_nemotronocr_language(options.lang) + # Resolve the request language. An empty `lang` list means "the + # engine's own default", which for nemotron-OCR is English. + native = self.resolve_ocr_languages() + language = native[0] if native else _NEMOTRON_OCR_ENGLISH # Initialize the model model_dir = self._resolve_model_dir(language, artifacts_path=artifacts_path) @@ -149,6 +140,24 @@ def __init__( lang=language, ) + def supported_ocr_languages(self) -> list[str]: + return ["en-Latn", "mul", *sorted(_NEMOTRON_OCR_MULTILINGUAL_TAGS)] + + def map_ocr_language(self, language: OcrLanguage) -> str: + if language.tag == "en-Latn": + return _NEMOTRON_OCR_ENGLISH + if language.is_multilingual or language.tag in _NEMOTRON_OCR_MULTILINGUAL_TAGS: + return _NEMOTRON_OCR_MULTILINGUAL + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + detail=( + "nemotron-OCR-v2 ships an English and a multilingual recognizer " + "only; use 'mul' to run the multilingual one." + ), + ) + @staticmethod def _fail_runtime(message: str) -> None: _log.error(message) diff --git a/docling/models/stages/ocr/ocr_mac_model.py b/docling/models/stages/ocr/ocr_mac_model.py index a388e61192..b7bc5e71f3 100644 --- a/docling/models/stages/ocr/ocr_mac_model.py +++ b/docling/models/stages/ocr/ocr_mac_model.py @@ -16,13 +16,55 @@ OcrOptions, ) from docling.datamodel.settings import settings +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_ocr_model import BaseOcrModel +from docling.utils.ocr_language import ( + OcrLanguage, + OcrLanguageResolver, + OcrLanguageSupport, +) from docling.utils.profiling import TimeRecorder _log = logging.getLogger(__name__) +# Recognition languages of a recent macOS, used when Vision cannot be queried. +# The real list is OS-version dependent, so it is only a fallback. +_OCRMAC_FALLBACK_LANGUAGES: tuple[str, ...] = ( + "en-US", + "fr-FR", + "it-IT", + "de-DE", + "es-ES", + "pt-BR", + "zh-Hans", + "zh-Hant", + "ko-KR", + "ja-JP", + "ru-RU", + "uk-UA", + "th-TH", + "vi-VT", +) + + +def _vision_recognition_languages() -> list[str]: + """The recognition languages the running macOS reports, or the fallback.""" + try: + import Vision + + # pyobjc exposes the ObjC classes dynamically, so ty cannot see them. + request = Vision.VNRecognizeTextRequest.alloc().init() # ty: ignore[unresolved-attribute] + languages, error = request.supportedRecognitionLanguagesAndReturnError_(None) + if error is None and languages: + return [str(language) for language in languages] + except Exception as exc: # pyobjc/Vision availability varies by OS version + _log.debug("Could not query Vision for recognition languages: %s", exc) + return list(_OCRMAC_FALLBACK_LANGUAGES) + class OcrMacModel(BaseOcrModel): + language_support = OcrLanguageSupport(multiple_languages=True) + def __init__( self, enabled: bool, @@ -40,6 +82,8 @@ def __init__( # multiplier for 72 dpi; the default 3.0 == 216 dpi. self.scale = self.options.scale + self._native_langs: list[str] = [] + self._vision_languages: list[str] = [] if self.enabled: if "darwin" != sys.platform: @@ -57,6 +101,32 @@ def __init__( self.reader_RIL = ocrmac.OCR + self._vision_languages = _vision_recognition_languages() + self._native_langs = self.resolve_ocr_languages() + + def supported_ocr_languages(self) -> list[str]: + return list(self._vision_languages) + + def map_ocr_language(self, language: OcrLanguage) -> str | list[str]: + if language.is_passthrough or language.is_multilingual: + # Vision has no script recognizers and no multilingual model; an + # empty `lang` list is how its own automatic behaviour is selected. + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + detail="Apple Vision needs explicit languages.", + ) + # Vision's own vocabulary is BCP-47 with regions, so match rather than map. + match = OcrLanguageResolver.match_ocr_language(language, self._vision_languages) + if match is None: + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + ) + return match + def __call__( self, conv_res: ConversionResult, page_batch: Iterable[Page] ) -> Iterable[Page]: @@ -91,7 +161,7 @@ def __call__( fname, recognition_level=self.options.recognition, framework=self.options.framework, - language_preference=self.options.lang, + language_preference=self._native_langs or None, ).recognize() im_width, im_height = high_res_image.size diff --git a/docling/models/stages/ocr/ppocr_languages.py b/docling/models/stages/ocr/ppocr_languages.py new file mode 100644 index 0000000000..2c6df45331 --- /dev/null +++ b/docling/models/stages/ocr/ppocr_languages.py @@ -0,0 +1,152 @@ +"""Canonical BCP-47 to PP-OCR recognizer tokens. + +RapidOCR and the KServe v2 OCR client both address PP-OCR recognizers by the +same tokens, so the mapping lives here rather than in either engine. RapidOCR +consults the installed `rapidocr` package for the authoritative PP-OCRv6 set and +falls back to the static copy below; the KServe client uses the static copy only, +so it never has to import `rapidocr`. + +The static token sets mirror the PP-OCR release notes summarised in +`docs/concepts/OCR.md`. They can drift from a newer `rapidocr`; this module is +their single owner. +""" + +from docling.utils.ocr_language import ( + OcrLanguage, + OcrLanguageResolver, +) + +# Recognition languages served by the PP-OCRv4 backbone (the torch fallback). +PPOCRV4_LANGS = frozenset( + {"arabic", "cyrillic", "devanagari", "ka", "korean", "latin", "ta", "te"} +) + +# Recognition languages served by the PP-OCRv5 backbone. +PPOCRV5_LANGS = frozenset( + { + "arabic", + "ch", + "cyrillic", + "devanagari", + "el", + "en", + "eslav", + "korean", + "latin", + "ta", + "te", + "th", + } +) + +# Static copy of the PP-OCRv6 recognition languages. RapidOCR prefers the set +# exported by the installed package; this is the offline/KServe fallback. +PPOCRV6_LANGS = frozenset( + { + "af", "az", "bs", "ca", "ch", "chinese_cht", "cs", "cy", "da", "de", + "en", "es", "et", "eu", "fi", "fr", "french", "ga", "german", "gl", + "hr", "hu", "id", "is", "it", "japan", "ku", "la", "lb", "lt", "lv", + "mi", "ms", "mt", "nl", "no", "oc", "pl", "pt", "qu", "rm", "ro", + "rs_latin", "sk", "sl", "sq", "sv", "sw", "tl", "tr", "uz", "vi", + } +) # fmt: skip + +# PP-OCR token used when `lang` is left empty: the engine's own default +# recognizer, which is Simplified Chinese. +PPOCR_DEFAULT_TOKEN = "ch" + +# Canonical tag -> PP-OCR token, for the languages whose token is not simply the +# primary subtag. `None` marks a tag that must *not* fall through to the generic +# rules below, because the token that looks right means something else. +_CANONICAL_TO_TOKEN: dict[str, str | None] = { + "zh-Hans": "ch", + "zh-Hant": "chinese_cht", + "ja-Jpan": "japan", + "ko-Kore": "korean", + "sr-Latn": "rs_latin", + # `tl` is PP-OCR's token; BCP-47 canonicalizes Tagalog to `fil`. + "fil-Latn": "tl", + # PP-OCR serves East Slavic with a narrower recognizer than `cyrillic`. + "ru-Cyrl": "eslav", + "uk-Cyrl": "eslav", + "be-Cyrl": "eslav", + # PP-OCR's `ka` is Kannada; BCP-47 `ka` is Georgian. + "kn-Knda": "ka", + "ka-Geor": None, +} + +# ISO 15924 script -> PP-OCR script-family token. Internal routing only: users +# name a language and this finds the script-wide recognizer that covers it, for +# the many languages PP-OCR serves no other way. +_SCRIPT_TO_TOKEN: dict[str, str] = { + "Latn": "latin", + "Cyrl": "cyrillic", + "Arab": "arabic", + "Deva": "devanagari", +} + +# Reverse of the language table, for rendering a vocabulary back as tags. The +# script recognizers are not reversed: they are named back as themselves, which +# is what the user types to select one. +_TOKEN_TO_CANONICAL: dict[str, list[str]] = {} +for _tag, _token in _CANONICAL_TO_TOKEN.items(): + if _token is not None: + _TOKEN_TO_CANONICAL.setdefault(_token, []).append(_tag) + +# PP-OCRv6 tokens that duplicate a language already reachable by its subtag. +_REDUNDANT_TOKENS = frozenset({"french", "german"}) + + +def ppocr_token(language: OcrLanguage, vocabulary: frozenset[str]) -> str | None: + """Map a canonical tag onto a PP-OCR token, or `None` if there is no model. + + `vocabulary` is the union of token sets the caller can actually reach, so + the resolution never returns a token the backend cannot serve. + """ + if language.is_passthrough: + # `arabic`, `cyrillic`: a recognizer named after a script, handed over as + # the user wrote it. + return language.native if language.native in vocabulary else None + if language.is_multilingual: + return None + + if language.tag in _CANONICAL_TO_TOKEN: + token = _CANONICAL_TO_TOKEN[language.tag] + return token if token is not None and token in vocabulary else None + + # The primary subtag identifies the recognizer only when the language is + # written in its usual script: PP-OCR's `az` and `uz` are the Latin ones. + if language.has_default_script and language.language in vocabulary: + return language.language + + # PP-OCR serves many languages only through a script-wide recognizer: there + # is no `ar` or `hi` model, and on the PP-OCRv4 backbone most of the + # vocabulary is script models. This routing is internal -- users name a + # language and docling finds the recognizer that covers it. + family = _SCRIPT_TO_TOKEN.get(language.script or "") + if family is not None and family in vocabulary: + return family + return None + + +def ppocr_supported_tags(vocabulary: frozenset[str], kind: str) -> list[str]: + """Render a PP-OCR token vocabulary back as the canonical tags it serves. + + `kind` is the calling engine's `OcrOptions.kind`: local RapidOCR and the + KServe v2 client address the same recognizers, and either one selects the + PP-OCR vocabulary. + """ + tags: set[str] = set() + for token in vocabulary: + if token in _REDUNDANT_TOKENS: + continue + if token in _TOKEN_TO_CANONICAL: + tags.update(_TOKEN_TO_CANONICAL[token]) + continue + try: + tags.add(OcrLanguageResolver.parse_ocr_language(token, kind).tag) + except ValueError: + # A token that is not a language code and has no reverse entry; + # it is unreachable from a canonical tag anyway. + continue + return sorted(tags) diff --git a/docling/models/stages/ocr/rapid_ocr_model.py b/docling/models/stages/ocr/rapid_ocr_model.py index 0fde1bb5a6..6e994df76a 100644 --- a/docling/models/stages/ocr/rapid_ocr_model.py +++ b/docling/models/stages/ocr/rapid_ocr_model.py @@ -16,8 +16,22 @@ RapidOcrOptions, ) from docling.datamodel.settings import settings +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_ocr_model import BaseOcrModel +from docling.models.stages.ocr.ppocr_languages import ( + PPOCR_DEFAULT_TOKEN, + PPOCRV4_LANGS, + PPOCRV5_LANGS, + PPOCRV6_LANGS, + ppocr_supported_tags, + ppocr_token, +) from docling.utils.accelerator_utils import decide_device +from docling.utils.ocr_language import ( + OcrLanguage, + OcrLanguageResolver, + OcrLanguageSupport, +) from docling.utils.profiling import TimeRecorder from docling.utils.utils import download_url_with_progress @@ -26,7 +40,8 @@ _log = logging.getLogger(__name__) -_RAPIDOCR_DEFAULT_LANGUAGE = "ch" +# Default OCR language as a canonical tag, for the prefetch entry points. +_RAPIDOCR_DEFAULT_LANGUAGE = "zh-Hans" # Recognition/detection model size for the PP-OCRv6 path; v4/v5 use "mobile". _RAPIDOCR_DET_MODEL_LANG = "ch" @@ -34,36 +49,11 @@ _RAPIDOCR_MODEL_TYPE = "small" _RAPIDOCR_V4V5_MODEL_TYPE = "mobile" -# Docling's default language names -> rapidocr language codes. -_DOCLING_LANG_NORMALIZE: dict[str, str] = {"chinese": "ch", "english": "en"} - # Inference backends docling supports. Must stay in sync with the `backend` Literal of # RapidOcrOptions in docling/datamodel/pipeline_options.py and with the mapping built in # _backend_to_engine_type(). _RAPIDOCR_BACKENDS: tuple[str, ...] = ("onnxruntime", "openvino", "paddle", "torch") -# Recognition languages served by the PP-OCRv4 backbone -_PPOCRV4_LANGS = frozenset( - {"arabic", "cyrillic", "devanagari", "ka", "korean", "latin", "ta", "te"} -) -# Recognition languages served by the PP-OCRv5 backbone -_PPOCRV5_LANGS = frozenset( - { - "arabic", - "ch", - "cyrillic", - "devanagari", - "el", - "en", - "eslav", - "korean", - "latin", - "ta", - "te", - "th", - } -) - @dataclass(frozen=True) class _RapidOcrArtifact: @@ -106,7 +96,7 @@ def _parse_rapidocr_model_spec(value: str) -> _RapidOcrModelSpec: if not separator or not backend or not lang or ":" in lang: raise ValueError( f"Invalid RapidOCR model spec {value!r}. " - "Expected ':', e.g. 'onnxruntime:th'." + "Expected ':', e.g. 'onnxruntime:th-Thai'." ) if backend not in _RAPIDOCR_BACKENDS: raise ValueError( @@ -115,7 +105,7 @@ def _parse_rapidocr_model_spec(value: str) -> _RapidOcrModelSpec: ) try: _resolve_rapidocr(lang, backend) - except ValueError as err: + except (ValueError, OcrLanguageNotSupportedError) as err: raise ValueError(f"Invalid RapidOCR model spec {value!r}: {err}") from err return _RapidOcrModelSpec(backend=backend, user_lang=lang) @@ -137,50 +127,73 @@ def _backend_to_engine_type(backend: str) -> "EngineType": return engine_types[backend] -def _resolve_rapidocr(lang: str, backend: str) -> _RapidOcrModelSpec: - """Map one requested language + backend onto a fully populated _RapidOcrModelSpec. +def _installed_ppocrv6_langs() -> frozenset[str]: + """The PP-OCRv6 recognition languages, from the installed rapidocr if present. - - Prefer PP-OCRv6 (whose recognizer is multilingual and covers ~52 codes) - - Otherwise fall back to PP-OCRv4 for the torch backend or PP-OCRv5 for the others. - - Raises when the language cannot be served by the resolved backbone. + Falls back to docling's static copy so the mapping stays usable for the + prefetcher and the KServe client, which do not require rapidocr. + """ + try: + from rapidocr.utils.model_resolver import PP_OCRV6_LANGS + except ImportError: + return PPOCRV6_LANGS + return frozenset(PP_OCRV6_LANGS) - Callers pass a single language; reducing a multi-language request is up to them. + +def _rapidocr_vocabulary(backend: str) -> frozenset[str]: + """PP-OCR tokens a backend can serve: v6 plus its own v4/v5 fallback set.""" + fallback = PPOCRV4_LANGS if backend == "torch" else PPOCRV5_LANGS + return _installed_ppocrv6_langs() | fallback + + +def _ppocr_version_for_token(token: str, backend: str) -> "OCRVersion": + """Which PP-OCR backbone serves a token on this backend. + + Prefers PP-OCRv6 (whose recognizer covers ~52 codes) and falls back to + PP-OCRv4 for the torch backend or PP-OCRv5 for the others. """ - from rapidocr.utils.model_resolver import COMMON_LANG_ALIASES, PP_OCRV6_LANGS from rapidocr.utils.typings import OCRVersion - code = lang.strip().lower() - code = _DOCLING_LANG_NORMALIZE.get(code, code) - aliased = COMMON_LANG_ALIASES.get(code, code) - - if aliased in PP_OCRV6_LANGS: - version = OCRVersion.PPOCRV6 - elif backend == "torch": - if aliased not in _PPOCRV4_LANGS: - raise ValueError( - f"RapidOCR torch backend does not support language {lang!r}. " - f"Supported: {sorted(PP_OCRV6_LANGS | _PPOCRV4_LANGS)}." - ) - version = OCRVersion.PPOCRV4 - elif aliased in _PPOCRV5_LANGS: - version = OCRVersion.PPOCRV5 - else: - raise ValueError( - f"RapidOCR {backend} backend does not support language {lang!r}. " - f"Supported: {sorted(PP_OCRV6_LANGS | _PPOCRV5_LANGS)}." + if token in _installed_ppocrv6_langs(): + return OCRVersion.PPOCRV6 + return OCRVersion.PPOCRV4 if backend == "torch" else OCRVersion.PPOCRV5 + + +def _resolve_rapidocr(lang: str, backend: str) -> _RapidOcrModelSpec: + """Map one language + backend onto a fully populated _RapidOcrModelSpec. + + `lang` may be a BCP-47 tag or one of PP-OCR's own tokens, matching what + `RapidOcrOptions.lang` accepts. + + Raises: + ValueError: `lang` is neither a PP-OCR token nor a valid BCP-47 tag. + OcrLanguageNotSupportedError: No PP-OCR recognizer serves it on `backend`. + + Callers pass a single language; reducing a multi-language request is up to them. + """ + language = OcrLanguageResolver.parse_ocr_language(lang, RapidOcrOptions.kind) + token = ppocr_token(language, _rapidocr_vocabulary(backend)) + if token is None: + raise OcrLanguageNotSupportedError( + f"RapidOCR (backend={backend})", + language.tag, + supported=ppocr_supported_tags( + _rapidocr_vocabulary(backend), RapidOcrOptions.kind + ), ) + version = _ppocr_version_for_token(token, backend) _log.debug( "RapidOCR resolved lang=%r backend=%r -> version=%s rec_lang=%r", lang, backend, version.value, - aliased, + token, ) return _RapidOcrModelSpec( backend=backend, user_lang=lang, - rapidocr_lang_token=aliased, + rapidocr_lang_token=token, ppocr_version=version, ) @@ -256,6 +269,8 @@ def _rapidocr_artifacts( class RapidOcrModel(BaseOcrModel): _model_repo_folder = "RapidOcr" + language_support = OcrLanguageSupport(multiple_languages=False) + def __init__( self, enabled: bool, @@ -273,6 +288,7 @@ def __init__( # multiplier for 72 dpi; the default 3.0 == 216 dpi. self.scale = self.options.scale + self._native_langs: list[str] = [] if self.enabled: try: @@ -293,23 +309,13 @@ def __init__( gpu_id = int(device.split(":")[1]) backend_enum = _backend_to_engine_type(self.options.backend) - # Reduce the user provided language list to one language + # One language, warn-and-truncate and coverage checks all happen here. + self._native_langs = self.resolve_ocr_languages() + rec_lang = self._native_langs[0] lang = ( - self.options.lang[0] - if self.options.lang - else _RAPIDOCR_DEFAULT_LANGUAGE + self.languages[0].tag if self.languages else _RAPIDOCR_DEFAULT_LANGUAGE ) - if len(self.options.lang) > 1: - _log.warning( - "RapidOCR uses a single language; using %r and ignoring %r.", - lang, - self.options.lang[1:], - ) - resolved: _RapidOcrModelSpec = _resolve_rapidocr(lang, self.options.backend) - assert resolved.ppocr_version is not None - assert resolved.rapidocr_lang_token is not None - ppocr_version = resolved.ppocr_version - rec_lang = resolved.rapidocr_lang_token + ppocr_version = _ppocr_version_for_token(rec_lang, self.options.backend) det_model_path = self.options.det_model_path cls_model_path = self.options.cls_model_path @@ -358,7 +364,7 @@ def __init__( ] if missing: listed = "\n".join(f" - {path}" for path in missing) - # `lang` is the user's own token, so the hint mirrors their config. + # `lang` is the canonical tag, which is what the prefetcher takes. raise FileNotFoundError( "RapidOCR artifacts not found or incomplete in artifacts_path.\n" f"Expected under: {target_dir}\n" @@ -451,6 +457,34 @@ def __init__( params=params, ) + def supported_ocr_languages(self) -> list[str]: + return ppocr_supported_tags( + _rapidocr_vocabulary(self.options.backend), RapidOcrOptions.kind + ) + + def resolve_ocr_languages(self) -> list[str]: + # An empty `lang` list means "the engine's own default", which for PP-OCR + # is the Simplified Chinese recognizer. + if not self.languages: + return [PPOCR_DEFAULT_TOKEN] + return super().resolve_ocr_languages() + + def map_ocr_language(self, language: OcrLanguage) -> str: + token = ppocr_token(language, _rapidocr_vocabulary(self.options.backend)) + if token is None: + raise OcrLanguageNotSupportedError( + f"RapidOCR (backend={self.options.backend})", + language.tag, + supported=self.supported_ocr_languages(), + detail=( + "RapidOCR has no multilingual recognizer; name the languages " + "explicitly." + ) + if language.is_multilingual + else None, + ) + return token + @classmethod def download_models( cls, diff --git a/docling/models/stages/ocr/tesseract_ocr_cli_model.py b/docling/models/stages/ocr/tesseract_ocr_cli_model.py index 3efa6a2ad7..4cf3cb29d8 100644 --- a/docling/models/stages/ocr/tesseract_ocr_cli_model.py +++ b/docling/models/stages/ocr/tesseract_ocr_cli_model.py @@ -22,12 +22,16 @@ TesseractCliOcrOptions, ) from docling.datamodel.settings import settings +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_ocr_model import BaseOcrModel -from docling.utils.ocr_utils import ( +from docling.models.stages.ocr.tesseract_utils import ( + map_tesseract_language, map_tesseract_script, parse_tesseract_orientation, tesseract_box_to_bounding_rectangle, + tesseract_language_to_tag, ) +from docling.utils.ocr_language import OcrLanguage, OcrLanguageSupport from docling.utils.profiling import TimeRecorder _log = logging.getLogger(__name__) @@ -37,6 +41,8 @@ class TesseractOcrCliModel(BaseOcrModel): + language_support = OcrLanguageSupport(multiple_languages=True) + def __init__( self, enabled: bool, @@ -59,7 +65,10 @@ def __init__( self._version: Optional[str] = None self._tesseract_languages: Optional[List[str]] = None self._script_prefix: Optional[str] = None - self._is_auto: bool = "auto" in self.options.lang + # No languages requested: Tesseract runs orientation and script + # detection per page and picks a `script/` reader from the result. + self._auto_script: bool = not self.languages + self._native_langs: List[str] = [] # Pre-validate and store sanitized subprocess arguments at construction time # so that all subsequent subprocess calls use only these already-validated values. @@ -69,10 +78,6 @@ def __init__( if self.options.path is not None else None ) - if self.options.lang: - for _lang_token in self.options.lang: - if _lang_token != "auto": - self._sanitize_lang(_lang_token) if self.enabled: try: @@ -87,6 +92,47 @@ def __init__( "Alternatively, Docling has support for other OCR engines. See the documentation." ) + if self._auto_script and "osd" not in (self._tesseract_languages or []): + raise ImportError( + "An empty OCR language list runs Tesseract's orientation and " + "script detection, which needs the 'osd' traineddata. Install " + "it (e.g. the tesseract-ocr-osd package) or name a language " + "explicitly in `ocr_options.lang`." + ) + + # Needs the installed language list, so it runs after the probe above. + self._native_langs = [ + self._sanitize_lang(lang) for lang in self.resolve_ocr_languages() + ] + + def supported_ocr_languages(self) -> List[str]: + tags = { + tag + for name in self._tesseract_languages or [] + if ( + tag := tesseract_language_to_tag( + name, self._script_prefix or "", TesseractCliOcrOptions.kind + ) + ) + } + return sorted(tags) + + def map_ocr_language(self, language: OcrLanguage) -> str | List[str]: + assert self._tesseract_languages is not None + name = map_tesseract_language(language, self._script_prefix or "") + if name is None or name not in self._tesseract_languages: + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + detail=( + f"No traineddata file {name!r} is installed." + if name is not None + else "Tesseract has no traineddata for it." + ), + ) + return name + @staticmethod def _sanitize_lang(lang: str) -> str: """Validate and sanitize a Tesseract language identifier to prevent argument injection. @@ -167,16 +213,14 @@ def _run_tesseract(self, ifilename: str, osd: Optional[pd.DataFrame]): Run tesseract CLI """ cmd = [self._safe_tesseract_cmd] - if self._is_auto and osd is not None: + if self._auto_script and osd is not None: lang = self._parse_language(osd) if lang is not None: cmd.append("-l") cmd.append(self._sanitize_lang(lang)) - elif self.options.lang is not None and len(self.options.lang) > 0: + elif self._native_langs: cmd.append("-l") - cmd.append( - "+".join(self._sanitize_lang(lang) for lang in self.options.lang) - ) + cmd.append("+".join(self._native_langs)) if self._safe_tessdata_path is not None: cmd.append("--tessdata-dir") @@ -330,7 +374,7 @@ def __call__( ) # Skipping if OSD fail when in auto mode, otherwise proceed # to OCR in the hope OCR will succeed while OSD failed - if self._is_auto: + if self._auto_script: continue if doc_orientation != 0: high_res_image = high_res_image.rotate( diff --git a/docling/models/stages/ocr/tesseract_ocr_model.py b/docling/models/stages/ocr/tesseract_ocr_model.py index 8862edc1d1..8c739eeaff 100644 --- a/docling/models/stages/ocr/tesseract_ocr_model.py +++ b/docling/models/stages/ocr/tesseract_ocr_model.py @@ -15,18 +15,24 @@ TesseractOcrOptions, ) from docling.datamodel.settings import settings +from docling.exceptions import OcrLanguageNotSupportedError from docling.models.base_ocr_model import BaseOcrModel -from docling.utils.ocr_utils import ( +from docling.models.stages.ocr.tesseract_utils import ( + map_tesseract_language, map_tesseract_script, parse_tesseract_orientation, tesseract_box_to_bounding_rectangle, + tesseract_language_to_tag, ) +from docling.utils.ocr_language import OcrLanguage, OcrLanguageSupport from docling.utils.profiling import TimeRecorder _log = logging.getLogger(__name__) class TesseractOcrModel(BaseOcrModel): + language_support = OcrLanguageSupport(multiple_languages=True) + def __init__( self, enabled: bool, @@ -41,11 +47,16 @@ def __init__( accelerator_options=accelerator_options, ) self.options: TesseractOcrOptions - self._is_auto: bool = "auto" in self.options.lang + # No languages requested: Tesseract runs orientation and script + # detection per page and picks a `script/` reader from the result. + self._auto_script: bool = not self.languages # multiplier for 72 dpi; the default 3.0 == 216 dpi. self.scale = self.options.scale self.reader = None self.script_readers: dict[str, tesserocr.PyTessBaseAPI] = {} + self._tesserocr_languages: list[str] = [] + self._native_langs: list[str] = [] + self.script_prefix = "" if self.enabled: install_errmsg = ( @@ -79,13 +90,23 @@ def __init__( # Initialize the tesseractAPI _log.debug("Initializing TesserOCR: %s", tesseract_version) - lang = "+".join(self.options.lang) - if any(lang.startswith("script/") for lang in self._tesserocr_languages): + if any(name.startswith("script/") for name in self._tesserocr_languages): self.script_prefix = "script/" else: self.script_prefix = "" + if self._auto_script and "osd" not in self._tesserocr_languages: + raise ImportError( + "An empty OCR language list runs Tesseract's orientation and " + "script detection, which needs the 'osd' traineddata. Install " + "it (e.g. the tesseract-ocr-osd package) or name a language " + "explicitly in `ocr_options.lang`." + ) + + # Needs the installed language list and the prefix, so it runs here. + self._native_langs = self.resolve_ocr_languages() + tesserocr_kwargs = { "init": True, "oem": tesserocr.OEM.DEFAULT, @@ -100,11 +121,12 @@ def __init__( main_psm = ( self.options.psm if self.options.psm is not None else tesserocr.PSM.AUTO ) - if lang == "auto": + if self._auto_script: + # No `lang`: the per-page OSD pass picks a script reader instead. self.reader = tesserocr.PyTessBaseAPI(psm=main_psm, **tesserocr_kwargs) else: self.reader = tesserocr.PyTessBaseAPI( - lang=lang, + lang="+".join(self._native_langs), psm=main_psm, **tesserocr_kwargs, ) @@ -114,6 +136,33 @@ def __init__( ) self.reader_RIL = tesserocr.RIL + def supported_ocr_languages(self) -> list[str]: + tags = { + tag + for name in self._tesserocr_languages + if ( + tag := tesseract_language_to_tag( + name, self.script_prefix, TesseractOcrOptions.kind + ) + ) + } + return sorted(tags) + + def map_ocr_language(self, language: OcrLanguage) -> str | list[str]: + name = map_tesseract_language(language, self.script_prefix) + if name is None or name not in self._tesserocr_languages: + raise OcrLanguageNotSupportedError( + self._engine_name, + language.tag, + supported=self.supported_ocr_languages(), + detail=( + f"No traineddata file {name!r} is installed." + if name is not None + else "Tesseract has no traineddata for it." + ), + ) + return name + def __del__(self): if self.reader is not None: # Finalize the tesseractAPI @@ -166,7 +215,7 @@ def __call__( ) # Skipping if OSD fail when in auto mode, otherwise proceed # to OCR in the hope OCR will succeed while OSD failed - if self._is_auto: + if self._auto_script: continue else: doc_orientation = parse_tesseract_orientation( @@ -176,7 +225,7 @@ def __call__( high_res_image = high_res_image.rotate( -doc_orientation, expand=True ) - if self._is_auto: + if self._auto_script: script = osd["script_name"] script = map_tesseract_script(script) lang = f"{self.script_prefix}{script}" diff --git a/docling/models/stages/ocr/tesseract_utils.py b/docling/models/stages/ocr/tesseract_utils.py new file mode 100644 index 0000000000..91bf82d353 --- /dev/null +++ b/docling/models/stages/ocr/tesseract_utils.py @@ -0,0 +1,147 @@ +"""Tessdata names, orientation and box geometry, shared by both Tesseract models. + +`tesseract_ocr_model.py` (the tesserocr bindings) and `tesseract_ocr_cli_model.py` +drive the same engine through different front-ends, so everything that is about +Tesseract itself rather than about either front-end lives here -- the same reason +`ppocr_languages.py` sits beside the two engines that speak PP-OCR. +""" + +from typing import Optional, Tuple + +import langcodes +from docling_core.types.doc import BoundingBox, CoordOrigin +from docling_core.types.doc.page import BoundingRectangle + +from docling.utils.ocr_language import ( + OcrLanguage, + OcrLanguageResolver, +) +from docling.utils.orientation import CLIPPED_ORIENTATIONS, rotate_bounding_box + + +def map_tesseract_script(script: str) -> str: + r"""Map an OSD-reported script name onto its tessdata `script/` file name.""" + if script == "Katakana" or script == "Hiragana": + script = "Japanese" + elif script == "Han": + script = "HanS" + elif script == "Korean": + script = "Hangul" + return script + + +# Canonical tag -> tessdata language file, where Tesseract deviates from +# ISO 639-2/T. Everything else is handled by `.to_alpha3(variant="T")`. +_TESSERACT_LANGUAGE_NAMES: dict[str, str] = { + "zh-Hans": "chi_sim", + "zh-Hant": "chi_tra", + "sr-Cyrl": "srp", + "sr-Latn": "srp_latn", + "az-Cyrl": "aze_cyrl", + "az-Latn": "aze", + "uz-Cyrl": "uzb_cyrl", + "uz-Latn": "uzb", + "ku-Latn": "kmr", + "nb-Latn": "nor", + "nn-Latn": "nor", + "no-Latn": "nor", + # Fraktur has its own traineddata; `to_alpha3()` would flatten it to `deu`. + "de-Latf": "deu_latf", +} + +_TESSERACT_TO_CANONICAL: dict[str, str] = { + name: tag for tag, name in _TESSERACT_LANGUAGE_NAMES.items() +} + + +def map_tesseract_language(language: OcrLanguage, script_prefix: str) -> str | None: + """Map a canonical tag onto a tessdata language file name. + + A passthrough names a traineddata file directly: `script/Cyrillic`. It is + always written with the `script/` prefix, but older tessdata installs list + the script files unprefixed, so the prefix is re-applied from what this + install actually reports. `mul` has no file of its own. + """ + if language.is_passthrough: + assert language.native is not None + name = language.native.removeprefix( + OcrLanguageResolver.TESSERACT_SCRIPT_FILE_PREFIX + ) + return f"{script_prefix}{name}" + if language.is_multilingual: + return None + if language.tag in _TESSERACT_LANGUAGE_NAMES: + return _TESSERACT_LANGUAGE_NAMES[language.tag] + # Tesseract's vocabulary *is* ISO 639-2/T: deu, fra, ell, ces, kat. + assert language.language is not None + return langcodes.Language.get(language.language).to_alpha3(variant="T") + + +def tesseract_language_to_tag(name: str, script_prefix: str, kind: str) -> str | None: + """Render one installed tessdata name back as a canonical tag. + + `kind` is the calling engine's `OcrOptions.kind`: the two Tesseract bindings + read the same tessdata names, and either one selects that vocabulary. + """ + if name in _TESSERACT_TO_CANONICAL: + return _TESSERACT_TO_CANONICAL[name] + if script_prefix and name.startswith(script_prefix): + # A script traineddata file is named back as itself: that is what the + # user has to type to select it. + return name + try: + return OcrLanguageResolver.parse_ocr_language(name, kind).tag + except ValueError: + return None + + +def parse_tesseract_orientation(orientation: str) -> int: + # Tesseract orientation is [0, 90, 180, 270] clockwise, bounding rectangle angles + # are [0, 360[ counterclockwise + parsed = int(orientation) + if parsed not in CLIPPED_ORIENTATIONS: + msg = ( + f"invalid tesseract document orientation {orientation}, " + f"expected orientation: {sorted(CLIPPED_ORIENTATIONS)}" + ) + raise ValueError(msg) + parsed = -parsed + parsed %= 360 + return parsed + + +def tesseract_box_to_bounding_rectangle( + bbox: BoundingBox, + *, + original_offset: Optional[BoundingBox] = None, + scale: float, + orientation: int, + im_size: Tuple[int, int], +) -> BoundingRectangle: + # box is in the top, left, height, width format, top left coordinates + rect = rotate_bounding_box(bbox, angle=orientation, im_size=im_size) + rect = BoundingRectangle( + r_x0=rect.r_x0 / scale, + r_y0=rect.r_y0 / scale, + r_x1=rect.r_x1 / scale, + r_y1=rect.r_y1 / scale, + r_x2=rect.r_x2 / scale, + r_y2=rect.r_y2 / scale, + r_x3=rect.r_x3 / scale, + r_y3=rect.r_y3 / scale, + coord_origin=CoordOrigin.TOPLEFT, + ) + if original_offset is not None: + if original_offset.coord_origin is not CoordOrigin.TOPLEFT: + msg = f"expected coordinate origin to be {CoordOrigin.TOPLEFT.value}" + raise ValueError(msg) + if original_offset is not None: + rect.r_x0 += original_offset.l + rect.r_x1 += original_offset.l + rect.r_x2 += original_offset.l + rect.r_x3 += original_offset.l + rect.r_y0 += original_offset.t + rect.r_y1 += original_offset.t + rect.r_y2 += original_offset.t + rect.r_y3 += original_offset.t + return rect diff --git a/docling/utils/model_downloader.py b/docling/utils/model_downloader.py index 167dc14d41..f8f9ba67a2 100644 --- a/docling/utils/model_downloader.py +++ b/docling/utils/model_downloader.py @@ -19,6 +19,7 @@ from docling.models.stages.ocr.easyocr_model import ( EasyOcrModel, _resolve_easyocr_recognition_models, + resolve_easyocr_languages, ) from docling.models.stages.ocr.nemotron_ocr_model import ( NemotronOcrModel, @@ -69,7 +70,7 @@ def download_models( with_rapidocr: bool = True, rapidocr_models: Optional[list[str]] = None, with_easyocr: bool = False, - easyocr_languages: Optional[list[str]] = None, + easyocr_languages: Optional[list[str]] = None, # BCP-47 tags with_nemotron_ocr: bool = False, ): if easyocr_languages is not None and not with_easyocr: @@ -80,7 +81,7 @@ def download_models( easyocr_recognition_models = ["english_g2", "latin_g2"] if easyocr_languages is not None: easyocr_recognition_models = _resolve_easyocr_recognition_models( - easyocr_languages + resolve_easyocr_languages(easyocr_languages) ) if output_dir is None: diff --git a/docling/utils/ocr_language.py b/docling/utils/ocr_language.py new file mode 100644 index 0000000000..49b85f7160 --- /dev/null +++ b/docling/utils/ocr_language.py @@ -0,0 +1,715 @@ +"""Canonicalization of OCR language requests to BCP-47 (RFC 5646). + +Docling exposes exactly one language vocabulary to users -- BCP-47 tags -- and +reduces every request to a `(language, script)` pair. Per-engine adapters +translate that pair into the engine's own notation (see +`docling.models.base_ocr_model.BaseOcrModel.map_ocr_language`). + +Region is discarded once it has inferred the script: `zh-CN` and `zh-Hans` are +the same recognizer, and `de-DE` vs `de-AT` is a distinction no OCR engine +docling supports can act on. + +`OcrLanguage` is the result type; `OcrLanguageResolver` owns the parsing itself +and the tables it consults. + +Users, however, arrive with the vocabulary of the engine they were already using +-- `ch` for RapidOCR, `chi_sim` for Tesseract, `ch_sim` for EasyOCR -- so each +options class also accepts its *own* engine's tokens, keyed by `OcrOptions.kind` +in `OcrLanguageResolver._NATIVE_VOCABULARIES`. Only tokens that plain BCP-47 +parsing gets wrong are listed there: the great majority of every engine's +vocabulary is ISO 639 already (`deu`, `fra`, `ru`, `ta`) and needs no entry. + +Nine of those tokens read as a *different language* under BCP-47 than their +engine means. For all nine the BCP-47 reading names a language that engine has +no model for -- no OCR engine docling supports ships a Chamorro, Georgian, +Frankish, Marshallese or Old English recognizer -- so letting the native reading +win takes nothing away, and `test_native_alias_never_shadows_a_reachable_language` +enforces exactly that. A user who genuinely wants the BCP-47 reading writes the +fully-qualified tag: the tables are keyed on the bare token, so `ch-Latn` +(Chamorro), `ka-Geor` (Georgian) and `ang-Latn` (Old English) bypass them. +""" + +import logging +from collections.abc import Mapping, Sequence +from functools import lru_cache +from types import MappingProxyType +from typing import ClassVar, NamedTuple + +import langcodes +from pydantic import BaseModel, ConfigDict + +_log = logging.getLogger(__name__) + +#: Multiple languages: the engine's broadest multilingual model. +MULTIPLE = "mul" + +_NO_TOKENS: Mapping[str, str] = MappingProxyType({}) + + +class _NativeVocabulary(NamedTuple): + """One engine family's own language codes, accepted alongside BCP-47. + + Attributes: + languages: Native token -> canonical tag, for the tokens plain BCP-47 + parsing would get wrong. + passthrough: Tokens naming a real model but no language -- PP-OCR's + script recognizers. They have no `(language, script)` form, so they + reach the engine verbatim rather than canonicalized. + script_files: tessdata `script/` file name -> ISO 15924 code, for + the engines that have such files. `None` for the engines that do not. + unrepresentable: Tokens docling refuses rather than mis-resolving, + mapped to the reason. Each names a model the canonical + `(language, script)` form cannot distinguish, so canonicalizing + would silently select a *different* recognizer. + """ + + languages: Mapping[str, str] + passthrough: frozenset[str] = frozenset() + script_files: Mapping[str, str] | None = None + unrepresentable: Mapping[str, str] = _NO_TOKENS + + +class OcrLanguageSupport(BaseModel): + """Static, engine-declared language capabilities. + + Attributes: + multiple_languages: Whether the engine can run several languages at + once. `False` marks a single-language engine, whose extra tags are + dropped with a warning. + """ + + model_config = ConfigDict(frozen=True) + + multiple_languages: bool = False + + +class OcrLanguage(BaseModel): + """One canonicalized OCR language request: a BCP-47 (language, script) pair. + + Attributes: + language: Primary subtag, lowercase. May be the reserved subtag `mul`. + script: ISO 15924 script code in title case. `None` only for the bare + reserved tags. + native: An engine's own token, kept verbatim, for the models no + `(language, script)` pair can name -- PP-OCR's script recognizers + (`arabic`, `cyrillic`) and Tesseract's `script/` files. Set + only for a passthrough, where it excludes `language` and `script` + and is what `tag` returns; `None` for every ordinary BCP-47 request. + """ + + model_config = ConfigDict(frozen=True) + + language: str | None = None + script: str | None = None + native: str | None = None + + @property + def tag(self) -> str: + """How this request is written back into `OcrOptions.lang`. + + A canonical BCP-47 tag (`de-Latn`, `mul`), or, for a passthrough, the + engine's own token verbatim. Returning the token unchanged is what keeps + `lang` idempotent: revalidating `["arabic"]` must not move it. + """ + if self.native is not None: + return self.native + return f"{self.language}-{self.script}" if self.script else self.language or "" + + @property + def is_passthrough(self) -> bool: + """An engine token that no `(language, script)` pair can express. + + PP-OCR's script recognizers (`arabic`, `cyrillic`) and Tesseract's + `script/` files: real models, named after a script rather than a + language, and handed to the engine untouched. + """ + return self.native is not None + + @property + def is_reserved(self) -> bool: + """One of the bare reserved tags, which must be requested alone.""" + return self.language in OcrLanguageResolver._RESERVED and self.script is None + + @property + def is_multilingual(self) -> bool: + return self.language == MULTIPLE + + @property + def has_default_script(self) -> bool: + """Whether `script` is the script CLDR considers likely for `language`. + + `de-Latn` and `en-Latn` do; `az-Cyrl` and `uz-Cyrl` do not. Engines use + this to decide whether the primary subtag alone still identifies the + right recognizer. + """ + if self.language is None or self.language in OcrLanguageResolver._RESERVED: + return False + return self.script == OcrLanguageResolver._default_script_for_language( + self.language + ) + + def __str__(self) -> str: + return self.tag + + +class OcrLanguageResolver: + """Parses user-supplied OCR language tokens into `OcrLanguage`. + + A namespace rather than an object: every entry point is a `@staticmethod`, + the vocabularies and legacy tables are class variables, and the expensive + steps memoize on their arguments alone. + """ + + # BCP-47's "undetermined". Docling does *not* accept it as an OCR language + # an empty `lang` list already says "let the engine decide" + _UNDETERMINED = "und" + + _RESERVED = frozenset({MULTIPLE}) + + _DOCS_URL = "https://docling-project.github.io/docling/concepts/OCR/" + + # Retired ways of asking an engine to decide for itself. An empty `lang` + # list says the same thing, so these point there rather than at a + # replacement tag. + _AUTO_TOKENS = frozenset({"auto", "osd", "und"}) + + # Engine tokens naming a script rather than a language. Reached only when + # the selected engine does not define them -- for PP-OCR they are real + # recognizers and resolve as passthroughs long before this. Docling has no + # engine-independent script family to redirect them to, so they get the same + # message `und-