feat: Canonicalize the OCR languages across all engines according to the BCP-47 standard

docling/.agents/skills/docling/references/cli.md

@@ -69,7 +69,7 @@ docling scan.pdf --ocr-engine tesserocr --output /tmp/   # needs system Tesserac
 docling scan.pdf --ocr-engine ocrmac --output /tmp/      # macOS Vision (mac only)
 docling scan.pdf --force-ocr --output /tmp/              # re-OCR even extractable text
 docling report.pdf --no-ocr --output /tmp/               # skip OCR (faster)
-docling scan.pdf --ocr-lang en --ocr-lang de --output /tmp/   # restrict languages
+docling scan.pdf --ocr-lang en,de --output /tmp/         # BCP-47 tags, comma-separated
 ```
 
 OCR engines are optional dependencies — see

docling/cli/export_utils.py

@@ -83,9 +83,13 @@ def _parse_page_range(raw: str | None) -> PageRange | None:
 
 
 def _split_list(raw: str | None) -> list[str] | None:
+    """Split a comma/semicolon-separated CLI value, dropping blanks.
+
+    Stripping matters: `--ocr-lang "en, de"` must yield `de`, not `" de"`.
+    """
     if raw is None:
         return None
-    return re.split(r"[;,]", raw)
+    return [item.strip() for item in re.split(r"[;,]", raw) if item.strip()]
 
 
 def _is_empty_output(path: Path) -> bool:

docling/cli/main.py

@@ -11,7 +11,7 @@
 from collections.abc import Iterable
 from enum import Enum
 from pathlib import Path
-from typing import Annotated, Literal, Type, cast
+from typing import Annotated, Any, Literal, Type, cast
 from urllib.parse import urlparse
 
 from docling.datamodel.service.responses import ChunkedDocumentResultItem
@@ -43,7 +43,7 @@
 from docling_core.transforms.visualizer.layout_visualizer import LayoutVisualizer
 from docling_core.types.doc import ImageRefMode
 from docling_core.utils.file import resolve_source_to_path
-from pydantic import TypeAdapter
+from pydantic import TypeAdapter, ValidationError
 from rich.console import Console
 
 from docling.cli.export_utils import (
@@ -161,6 +161,17 @@
 from docling.models.factories.base_factory import BaseFactory
 from docling.utils.profiling import ProfilingItem
 
+
+def _first_error_message(err: ValidationError) -> str:
+    """The most useful line of a pydantic error, for a typer.BadParameter."""
+    errors = err.errors()
+    if not errors:
+        return str(err)
+    message = errors[0].get("msg", "")
+    # Pydantic prefixes messages raised from a validator with "Value error, ".
+    return message.removeprefix("Value error, ") or str(err)
+
+
 warnings.filterwarnings(action="ignore", category=UserWarning, module="pydantic|torch")
 warnings.filterwarnings(action="ignore", category=FutureWarning, module="easyocr")
 
@@ -893,7 +904,17 @@ def convert(  # noqa: C901
         str | None,
         typer.Option(
             ...,
-            help="Provide a comma-separated list of languages used by the OCR engine. Note that each OCR engine has different values for the language names.",
+            help=(
+                "Comma-separated list of OCR languages as BCP-47 tags, e.g. "
+                "'en,de' or 'zh-Hant'. The selected engine's own language codes "
+                "are accepted too and mean what that engine means by them, so "
+                "'--ocr-engine rapidocr --ocr-lang ch' is Simplified Chinese. "
+                "Omit the option for the engine's default languages, or pass an "
+                "empty value (--ocr-lang '') to let the engine choose, which for "
+                "Tesseract is per-page script detection. 'mul' selects a "
+                "multilingual model on the engines that ship one; to skip OCR "
+                "entirely use --no-ocr."
+            ),
         ),
     ] = None,
     psm: Annotated[
@@ -1234,14 +1255,21 @@ def convert(  # noqa: C901
             resolved_ocr_mode = OcrMode.FULL_PAGE
         else:
             resolved_ocr_mode = ocr_mode
-        ocr_options: OcrOptions = ocr_factory.create_options(  # type: ignore
-            kind=ocr_engine,
-            mode=resolved_ocr_mode,
-        )
-
+        ocr_kwargs: dict[str, Any] = {"mode": resolved_ocr_mode}
         ocr_lang_list = _split_list(ocr_lang)
+        # `_split_list` returns None only when the option was not given, so an
+        # explicitly empty value reaches the engine as `lang=[]`: "your default".
         if ocr_lang_list is not None:
-            ocr_options.lang = ocr_lang_list
+            ocr_kwargs["lang"] = ocr_lang_list
+        try:
+            ocr_options: OcrOptions = ocr_factory.create_options(  # type: ignore
+                kind=ocr_engine,
+                **ocr_kwargs,
+            )
+        except ValidationError as err:
+            raise typer.BadParameter(
+                _first_error_message(err), param_hint="--ocr-lang"
+            ) from err
         if psm is not None and isinstance(
             ocr_options, TesseractOcrOptions | TesseractCliOcrOptions
         ):

docling/cli/models.py

@@ -30,7 +30,7 @@
 
 from docling.datamodel.settings import settings
 from docling.models.stages.ocr.easyocr_model import (
-    _resolve_easyocr_recognition_models,
+    resolve_easyocr_codes,
 )
 from docling.models.stages.ocr.rapid_ocr_model import _parse_rapidocr_model_spec
 from docling.models.utils.hf_model_download import download_hf_model
@@ -122,7 +122,12 @@ def download(
         typer.Option(
             ...,
             "--easyocr-lang",
-            help="EasyOCR language code to prefetch. Repeat for multiple languages.",
+            help=(
+                "OCR language to prefetch for EasyOCR, as a BCP-47 tag "
+                "(e.g. 'de', 'zh-Hant', 'ru'). EasyOCR's own codes are accepted "
+                "too and mean what EasyOCR means by them, so 'ch_sim' is "
+                "Simplified Chinese. Repeat for multiple."
+            ),
         ),
     ] = None,
     rapidocr_backend_lang: Annotated[
@@ -132,7 +137,11 @@ def download(
             "--rapidocr-backend-lang",
             help=(
                 "RapidOCR checkpoint set to prefetch, as '<backend>:<lang>' "
-                "(e.g. 'onnxruntime:el', 'torch:korean'). Repeat for multiple. Replaces the default set."
+                "with a BCP-47 language (e.g. 'onnxruntime:el', 'torch:ko'). "
+                "PP-OCR's own codes are accepted too, including its script "
+                "recognizers, which no language tag can name: "
+                "'onnxruntime:cyrillic', 'torch:ch'. Repeat for multiple. "
+                "Replaces the default set."
             ),
         ),
     ] = None,
@@ -156,7 +165,7 @@ def download(
                 param_hint="--easyocr-lang",
             )
         try:
-            _resolve_easyocr_recognition_models(easyocr_lang)
+            resolve_easyocr_codes(easyocr_lang)
         except ValueError as error:
             raise typer.BadParameter(str(error), param_hint="--easyocr-lang") from error
     if rapidocr_backend_lang is not None:

docling/cli/remote.py

@@ -36,9 +36,33 @@
     DoclingServiceClient,
     StatusWatcherKind,
 )
+from docling.utils.ocr_language import OcrLanguageResolver
 
 _log = logging.getLogger(__name__)
 
+
+def _canonicalize_ocr_lang(raw: Optional[str]) -> Optional[list[str]]:
+    """Canonicalize --ocr-lang locally, so a typo fails here and not remotely.
+
+    This command has no `--ocr-engine`, so it cannot pick an engine's own
+    vocabulary and uses the engine-independent one: native tokens that every
+    engine agrees on are accepted, and the handful that clash with a BCP-47 tag
+    of a different language are not.
+    """
+    # `_split_list` returns None only when the option was not given, so an
+    # explicitly empty value survives as `[]`: "let the engine choose".
+    tags = _split_list(raw)
+    if tags is None:
+        return None
+    try:
+        return [
+            language.tag
+            for language in OcrLanguageResolver.canonicalize_ocr_languages(tags)
+        ]
+    except ValueError as err:
+        raise typer.BadParameter(str(err), param_hint="--ocr-lang") from err
+
+
 _REMOTE_HELP = """\
 Convert documents through a remote docling-serve service instead of locally.
 
@@ -191,7 +215,17 @@ def convert_remote(
     ocr_lang: Annotated[
         Optional[str],
         typer.Option(
-            help="Comma-separated list of OCR languages (engine-specific names).",
+            help=(
+                "Comma-separated list of OCR languages as BCP-47 tags, e.g. "
+                "'en,de' or 'zh-Hant'. Widely-understood engine names such as "
+                "'chinese' or 'japan' are accepted too, but engine-specific ones "
+                "such as 'ch' are not, because this command does not choose the "
+                "engine. Omit the option for the service's default languages, or "
+                "pass an empty value (--ocr-lang '') to let the engine choose. "
+                "Canonicalized locally before the request is sent, so the "
+                "remote service must be recent enough to speak BCP-47 OCR "
+                "languages."
+            ),
         ),
     ] = None,
     enrich_code: Annotated[
@@ -309,7 +343,7 @@ def convert_remote(
         "to_formats": to_formats,
         "do_ocr": ocr,
         "force_ocr": force_ocr,
-        "ocr_lang": _split_list(ocr_lang),
+        "ocr_lang": _canonicalize_ocr_lang(ocr_lang),
         "do_table_structure": tables,
         "pipeline": pipeline,
         "do_code_enrichment": enrich_code,