From 7f2e3ad77e43349ca2afcb0db21a279c83a11577 Mon Sep 17 00:00:00 2001 From: David Riccitelli Date: Tue, 8 Sep 2026 13:23:48 +0300 Subject: [PATCH 1/2] fix: transliterate canonical IDs using account language --- .github/workflows/ci.yml | 17 +- CHANGELOG.md | 11 ++ README.md | 6 + docs/canonical_id_policy.md | 42 +++++ docs/packaging_slices_v7.md | 43 ++++- poetry.lock | 18 +- pyproject.toml | 3 + .../processors/test_id_allocator.py | 30 ++++ .../processors/test_id_transliteration.py | 156 ++++++++++++++++++ .../postprocessors/processors/test_slug.py | 86 ++++++++++ .../postprocessors/test_postprocessors.py | 64 +++++++ tests/kg_build/test_protocol.py | 20 +++ tests/tools/run_slice_smoke_imports.py | 9 +- .../kg_build/postprocessors/graph_io.py | 1 + .../kg_build/postprocessors/oneshot.py | 5 +- .../postprocessors/processors/id_allocator.py | 27 +-- .../postprocessors/processors/id_generator.py | 112 +++++++++---- .../processors/id_postprocessor.py | 7 +- .../postprocessors/processors/slug.py | 86 ++++++++++ wordlift_sdk/kg_build/protocol.py | 8 +- 20 files changed, 698 insertions(+), 53 deletions(-) create mode 100644 tests/kg_build/postprocessors/processors/test_id_transliteration.py create mode 100644 tests/kg_build/postprocessors/processors/test_slug.py create mode 100644 wordlift_sdk/kg_build/postprocessors/processors/slug.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 374b0bb..650369e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ permissions: jobs: slice-tests: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -51,6 +51,13 @@ jobs: - name: Install Poetry run: ~/.local/bin/pipx install poetry==2.2.1 + - name: Install ICU 74.2 development libraries + if: matrix.slice == 'kg-build' + run: | + sudo apt-get update + sudo apt-get install -y libicu-dev pkg-config g++ + test "$(pkg-config --modversion icu-i18n)" = "74.2" + - name: Install slice dependencies run: poetry install --no-interaction --extras "${{ matrix.slice }}" @@ -65,7 +72,7 @@ jobs: run: poetry run python tests/tools/run_slice_tests.py "${{ matrix.slice }}" -q test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository @@ -86,6 +93,12 @@ jobs: - name: Install Poetry run: ~/.local/bin/pipx install poetry==2.2.1 + - name: Install ICU 74.2 development libraries + run: | + sudo apt-get update + sudo apt-get install -y libicu-dev pkg-config g++ + test "$(pkg-config --modversion icu-i18n)" = "74.2" + - name: Install dependencies run: poetry install --no-interaction --extras "all" diff --git a/CHANGELOG.md b/CHANGELOG.md index 6744bda..a19266f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## Unreleased + +### Changed + +- KG build canonical IDs use language-aware ASCII transliteration from + `account.language`, with stable source hashes for non-ASCII names without URLs. + German umlauts use `ae`/`oe`/`ue`; unsupported readings use a hashed fallback. + Generated IDs can change; existing root lookup mappings remain authoritative. +- The `kg-build` and `all` extras require PyICU 2.16.2 and native ICU 74.2 for + reproducible transliteration. See `docs/packaging_slices_v7.md` for setup. + ## 8.4.3 - 2026-08-19 ### Fixed diff --git a/README.md b/README.md index 7d24254..4a2bb1d 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,12 @@ pip install "wordlift-sdk[all]" Requires Python 3.10–3.14. +The `kg-build` and `all` extras also require native ICU 74.2 development +libraries and a C++ compiler to build PyICU. See the +[installation instructions](docs/packaging_slices_v7.md#native-icu-for-kg-build). +Canonical IDs use account-language-aware ASCII transliteration; see the +[ID policy and migration notes](docs/canonical_id_policy.md#language-aware-ascii-identifiers). + `wordlift-sdk` v7 uses a lean base package plus optional extras. The import namespace remains `wordlift_sdk.*`; feature packages load lazily and raise an install hint if you access an export without the matching extra installed. diff --git a/docs/canonical_id_policy.md b/docs/canonical_id_policy.md index 34ac491..baefa00 100644 --- a/docs/canonical_id_policy.md +++ b/docs/canonical_id_policy.md @@ -87,3 +87,45 @@ Behavior: - duplicate URL rows in dataframe lookup resolve to the shortest IRI path depth (tie-break: shorter full IRI, then first row order) - lookup misses fall back to normal canonical ID generation + +## Language-aware ASCII Identifiers + +The canonical generator and `IdAllocator` share ASCII slug normalization. +Cloud callbacks and both postprocessor worker modes derive language from +`context.account.language`; direct callers can pass the optional `language` +argument. Language tags are case-insensitive and accept regional forms such +as `de-DE` or `de_DE`. Missing language uses Latin accent transliteration only. + +- German applies `ä → ae`, `ö → oe`, `ü → ue`, `ß → ss` (including capitals). + Other languages retain the generic Latin behavior, such as `ü → u`. +- Russian, Ukrainian, Greek, Arabic, Hebrew, Korean Hangul, and Hindi use + explicit ICU romanization routes. Mandarin Chinese uses `Han-Latin`. +- Japanese transliterates hiragana and katakana. Kanji, Cantonese, unknown + scripts, and scripts outside the selected language route do not receive a + guessed pronunciation. Unsupported characters are omitted from the readable + slug; an empty result uses `thing`. +- Inputs are normalized to NFC, so canonically equivalent composed and + decomposed text generates the same identifier. Existing punctuation, + separators, ASCII-only behavior, and parent nesting conventions remain. + +For non-ASCII names without a URL, the readable slug is suffixed with the full +SHA-256 digest of the NFC-normalized, stripped, lowercase original name +(before transliteration), even when romanization succeeds. This prevents distinct +names with the same romanization from merging +across separate callback graphs. Unsupported names use `thing-`. +Names with a URL keep the existing URL-hash suffix. Identical names without +another identity signal retain their existing ambiguity. +Source-hash identities do not acquire positional sibling suffixes; graph-local +collision suffixes still keep repeated identical names separate. This prevents +sorting newly generated IRIs from changing distinct sibling IDs on a later pass. + +Existing authoritative root IRI lookups take precedence over generation. +The fallback pass preserves already canonical dataset IRIs, but explicitly +handled roots and dependents can be regenerated. Changing account language or +regenerating an older non-ASCII identifier can create a new entity, including +when the URL hash is unchanged. This change does not migrate or delete existing +entities automatically: retain lookup mappings when existing root identity +must be preserved. + +Transliteration uses PyICU 2.16.2 and native ICU 74.2; installation details are +in [Packaging Slices](packaging_slices_v7.md#native-icu-for-kg-build). diff --git a/docs/packaging_slices_v7.md b/docs/packaging_slices_v7.md index 7bc469f..5cd263d 100644 --- a/docs/packaging_slices_v7.md +++ b/docs/packaging_slices_v7.md @@ -71,7 +71,7 @@ distribution model: - Modules: `wordlift_sdk.kg_build.*` - Dependencies: `advertools`, `gql`, `google-auth`, `gspread`, `jinja2`, `lxml`, `morph-kgc`, `worph`, `pandas`, `playwright`, `pydantic-core`, `pyshacl`, - `python-liquid`, `rdflib`, `requests`, `tomli`, `tqdm`, `trafilatura` + `PyICU`, `python-liquid`, `rdflib`, `requests`, `tomli`, `tqdm`, `trafilatura` - Notes: this is intentionally broad because `kg_build` composes multiple subsystems. @@ -82,6 +82,47 @@ distribution model: - `all` - Installs every optional dependency declared above. +## Native ICU for kg-build + +Only `kg-build` and `all` install `PyICU==2.16.2`. PyICU builds from source and +requires ICU development headers/libraries, `pkg-config`, and a C++ compiler. +The canonical transliteration baseline is ICU **74.2**. Pin the native ICU +version as well as PyICU in deployment images: ICU data changes can otherwise +change generated IDs. Non-ASCII canonicalization rejects a different linked +ICU version with an actionable error; ASCII-only normalization is unchanged. +CI uses Ubuntu 24.04 and checks this version explicitly. + +On Ubuntu 24.04: + +```bash +sudo apt-get update +sudo apt-get install -y libicu-dev pkg-config g++ +test "$(pkg-config --modversion icu-i18n)" = "74.2" +uv pip install "wordlift-sdk[kg-build]" +``` + +On other platforms, provision ICU 74.2 and set `PKG_CONFIG_PATH` to its +`lib/pkgconfig` directory before installing the extra. A newer system ICU is +not an equivalent replacement for reproducible canonical IDs. For example, +with an existing ICU 74.2 installation under `/opt/local` on macOS: + +```bash +export PKG_CONFIG_PATH="/opt/local/lib/pkgconfig" +test "$(pkg-config --modversion icu-i18n)" = "74.2" +uv pip install "wordlift-sdk[kg-build]" +``` + +If PyICU was already compiled against another ICU, rebuild it after selecting +the 74.2 `PKG_CONFIG_PATH` (`uv pip install --reinstall --no-cache pyicu==2.16.2`). + +Postprocessor interpreters configured separately must install the same extra +and native ICU version. Check the linked runtime after installation: + +```bash +python -c 'import icu; print(icu.VERSION, icu.ICU_VERSION)' +# 2.16.2 74.2 +``` + ## Boundary Rules - Package `__init__` modules must stay lazy so importing `wordlift_sdk` or a diff --git a/poetry.lock b/poetry.lock index d6c4de1..894a707 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3040,6 +3040,18 @@ files = [ [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pyicu" +version = "2.16.2" +description = "Python extension wrapping the ICU C++ API" +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"kg-build\" or extra == \"all\"" +files = [ + {file = "pyicu-2.16.2.tar.gz", hash = "sha256:006d51e24b5ec76df6ec2130f3dde269c51db8b8cfebb7d45a427dde0d10aa52"}, +] + [[package]] name = "pyopenssl" version = "26.0.0" @@ -4617,13 +4629,13 @@ test = ["coverage[toml]", "zope.event", "zope.testing"] testing = ["coverage[toml]", "zope.event", "zope.testing"] [extras] -all = ["advertools", "google-auth", "gql", "gspread", "jinja2", "lxml", "morph-kgc", "pandas", "playwright", "pycountry", "pydantic-core", "pyoxigraph", "pyshacl", "python-liquid", "rdflib", "requests", "tomli", "tqdm", "trafilatura", "twisted", "worph"] +all = ["advertools", "google-auth", "gql", "gspread", "jinja2", "lxml", "morph-kgc", "pandas", "playwright", "pycountry", "pydantic-core", "pyicu", "pyoxigraph", "pyshacl", "python-liquid", "rdflib", "requests", "tomli", "tqdm", "trafilatura", "twisted", "worph"] core = [] google-search-console = ["google-auth", "pandas", "pycountry", "tqdm", "twisted"] google-sheets = ["google-auth", "gspread", "pandas"] graph = ["pyoxigraph", "pyshacl", "python-liquid", "rdflib", "requests", "tomli", "tqdm"] ingestion = ["advertools", "google-auth", "gspread", "lxml", "morph-kgc", "pandas", "playwright", "pyshacl", "rdflib", "requests", "tqdm", "trafilatura", "worph"] -kg-build = ["advertools", "google-auth", "gql", "gspread", "jinja2", "lxml", "morph-kgc", "pandas", "playwright", "pydantic-core", "pyshacl", "python-liquid", "rdflib", "requests", "tomli", "tqdm", "trafilatura", "worph"] +kg-build = ["advertools", "google-auth", "gql", "gspread", "jinja2", "lxml", "morph-kgc", "pandas", "playwright", "pydantic-core", "pyicu", "pyshacl", "python-liquid", "rdflib", "requests", "tomli", "tqdm", "trafilatura", "worph"] legacy = ["google-auth", "gql", "gspread", "lxml", "pandas", "playwright", "pycountry", "pydantic-core", "python-liquid", "rdflib", "requests", "tqdm", "twisted"] render = ["lxml", "playwright"] structured-data = ["advertools", "lxml", "morph-kgc", "playwright", "pyshacl", "rdflib", "requests", "tqdm", "worph"] @@ -4633,4 +4645,4 @@ workflow = ["advertools", "google-auth", "gql", "gspread", "lxml", "pandas", "pl [metadata] lock-version = "2.1" python-versions = ">=3.10, <3.15" -content-hash = "731b2f505018d535f52f10fef303b9530f5957abe1f2f3700dad81a606d55b70" +content-hash = "0cece997bf0124753b675c8b32f1dcc0a26b8ba6f4fddb9ada32042cfc2215f5" diff --git a/pyproject.toml b/pyproject.toml index f12a950..45141ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ gspread = { version = "^6.1.2", optional = true } google-auth = { version = "^2.35.0", optional = true } tqdm = { version = "^4.67.1", optional = true } advertools = { version = "^0.17.1", optional = true } +pyicu = { version = "2.16.2", optional = true } pycountry = { version = "^26.0.0", optional = true } python-liquid = { version = "^2.0.1", optional = true } jinja2 = { version = "^3.1.6", optional = true } @@ -117,6 +118,7 @@ kg-build = [ "tomli", "tqdm", "trafilatura", + "pyicu", ] legacy = [ "gql", @@ -155,6 +157,7 @@ all = [ "tqdm", "trafilatura", "twisted", + "pyicu", ] [tool.poetry.group.dev.dependencies] diff --git a/tests/kg_build/postprocessors/processors/test_id_allocator.py b/tests/kg_build/postprocessors/processors/test_id_allocator.py index c420626..4eb85b7 100644 --- a/tests/kg_build/postprocessors/processors/test_id_allocator.py +++ b/tests/kg_build/postprocessors/processors/test_id_allocator.py @@ -1,5 +1,8 @@ from __future__ import annotations +import hashlib +import pytest + from rdflib import Graph, Literal, RDF, URIRef import wordlift_sdk.kg_build.postprocessors.processors.id_allocator as id_allocator_module @@ -107,3 +110,30 @@ def test_swap_iri_and_helpers() -> None: assert IdAllocator._first_value(g, new, "name") == "A" assert IdAllocator._base_from_priority(g, new) == "A" + + +@pytest.mark.parametrize( + "name,language,slug", + [("Müller", "de", "mueller"), ("Müller", "tr", "muller"), ("東京", "ja", "thing")], +) +def test_multilingual_allocator_identity_and_url_hash(name, language, slug) -> None: + allocator = IdAllocator("https://data.example.com/dataset", language=language) + graph = Graph() + digest = hashlib.sha256(name.lower().encode("utf-8")).hexdigest() + expected = URIRef(f"https://data.example.com/dataset/things/{slug}-{digest}") + assert allocator.new_independent(graph, base_value=name) == expected + subject = URIRef("https://example.com/source") + graph.add((subject, URIRef("http://schema.org/name"), Literal(name))) + assert allocator.assign(graph, subject) == expected + assert allocator.assign(graph, expected) == expected + url = "https://example.com/page?b=2&a=1#fragment" + result = allocator.new_independent(Graph(), base_value=name, url_value=url) + assert str(result).endswith(f"/{slug}-{allocator._url_hash(url)}") + child = allocator.new_child(Graph(), parent=expected, base_value=name) + assert str(child) == f"{expected}/things/{slug}-{digest}" + assert ( + allocator.new_child( + Graph(), parent=expected, base_value=name, force_index=True, index=7 + ) + == child + ) diff --git a/tests/kg_build/postprocessors/processors/test_id_transliteration.py b/tests/kg_build/postprocessors/processors/test_id_transliteration.py new file mode 100644 index 0000000..cf2a09c --- /dev/null +++ b/tests/kg_build/postprocessors/processors/test_id_transliteration.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import hashlib + +import pytest +from rdflib import Graph, Literal, RDF, URIRef + +from wordlift_sdk.kg_build.postprocessors.processors.id_generator import ( + CanonicalIdGenerator, +) + +SCHEMA = "http://schema.org/" +DATASET = "https://data.example.com" + + +def _entity(graph: Graph, key: str, name: str, type_name: str = "WebPage") -> URIRef: + subject = URIRef(f"https://example.com/{key}") + graph.add((subject, RDF.type, URIRef(f"{SCHEMA}{type_name}"))) + graph.add((subject, URIRef(f"{SCHEMA}name"), Literal(name))) + return subject + + +@pytest.mark.parametrize("strategy", ["legacy", "dependency_graph"]) +def test_non_latin_pages_remain_distinct_and_repeatable(strategy: str) -> None: + graph = Graph() + for key, name in (("a", "東京"), ("b", "北京")): + _entity(graph, key, name) + generator = CanonicalIdGenerator(strategy=strategy) + generator.apply(graph, DATASET) + subjects = set(graph.subjects(RDF.type, URIRef(f"{SCHEMA}WebPage"))) + assert len(subjects) == 2 + assert all(str(subject).isascii() for subject in subjects) + assert len(graph) == 4 + before = set(graph) + generator.apply(graph, DATASET) + assert set(graph) == before + + +@pytest.mark.parametrize("strategy", ["legacy", "dependency_graph"]) +@pytest.mark.parametrize("type_name", ["WebPage", "Product", "Person"]) +def test_transliteration_collisions_stay_distinct_across_graphs( + strategy: str, type_name: str +) -> None: + generator = CanonicalIdGenerator(strategy=strategy) + subjects = [] + for name in ("Müller", "Mueller"): + graph = Graph() + _entity(graph, "same-source", name, type_name) + generator.apply(graph, DATASET, language="de") + subjects.append(next(graph.subjects(RDF.type, URIRef(f"{SCHEMA}{type_name}")))) + digest = hashlib.sha256("müller".encode()).hexdigest() + assert str(subjects[0]).endswith(f"/mueller-{digest}") + assert str(subjects[1]).endswith("/mueller") + assert subjects[0] != subjects[1] + + +@pytest.mark.parametrize("strategy", ["legacy", "dependency_graph"]) +def test_language_changes_do_not_leak_between_calls_and_keep_url_hash( + strategy: str, +) -> None: + generator = CanonicalIdGenerator(strategy=strategy) + url = "https://example.com/page" + digest = hashlib.sha256(url.encode()).hexdigest() + for language, slug in (("de", "mueller"), ("tr", "muller"), ("de", "mueller")): + graph = Graph() + subject = _entity(graph, "a", "Müller") + graph.add((subject, URIRef(f"{SCHEMA}url"), Literal(url))) + generator.apply(graph, DATASET, language=language) + assert URIRef(f"{DATASET}/web-pages/{slug}-{digest}") in set(graph.subjects()) + + +@pytest.mark.parametrize("strategy", ["legacy", "dependency_graph"]) +def test_lookup_iri_remains_authoritative(strategy: str) -> None: + expected = URIRef(f"{DATASET}/web-pages/Müller") + + class Lookup: + def iri_for_subject(self, graph: Graph, subject: URIRef) -> str: + return str(expected) + + graph = Graph() + _entity(graph, "a", "Müller") + generator = CanonicalIdGenerator(strategy=strategy) + generator.apply(graph, DATASET, iri_lookup=Lookup(), language="de") + assert set(graph.subjects()) == {expected} + + +@pytest.mark.parametrize("strategy", ["legacy", "dependency_graph"]) +def test_actions_and_questions_receive_language(strategy: str) -> None: + graph = Graph() + root = _entity(graph, "page", "Page") + action = _entity(graph, "action", "Prüfen", "Action") + faq = _entity(graph, "faq", "FAQ", "FAQPage") + question = _entity(graph, "question", "Für wen", "Question") + graph.add((root, URIRef(f"{SCHEMA}potentialAction"), action)) + graph.add((root, URIRef(f"{SCHEMA}hasPart"), faq)) + graph.add((faq, URIRef(f"{SCHEMA}mainEntity"), question)) + CanonicalIdGenerator(strategy=strategy).apply(graph, DATASET, language="de") + for type_name, original, slug in ( + ("Action", "prüfen", "pruefen"), + ("Question", "für wen", "fuer-wen"), + ): + subject = next(graph.subjects(RDF.type, URIRef(f"{SCHEMA}{type_name}"))) + digest = hashlib.sha256(original.encode()).hexdigest() + assert str(subject).endswith(f"/{slug}-{digest}") + + +@pytest.mark.parametrize("strategy", ["legacy", "dependency_graph"]) +@pytest.mark.parametrize("type_name", ["Question", "ImageObject", "VideoObject"]) +@pytest.mark.parametrize("names", [("大阪", "東京"), ("大阪", "大阪", "大阪")]) +def test_unicode_siblings_remain_distinct_and_idempotent( + strategy: str, type_name: str, names: tuple[str, ...] +) -> None: + graph = Graph() + parent = _entity(graph, "page", "Page") + predicate = { + "Question": "mainEntity", + "ImageObject": "image", + "VideoObject": "video", + }[type_name] + if type_name == "Question": + faq = _entity(graph, "faq", "FAQ", "FAQPage") + graph.add((parent, URIRef(f"{SCHEMA}hasPart"), faq)) + parent = faq + for key, name in zip(("a", "b", "c"), names): + child = _entity(graph, key, name, type_name) + graph.add((parent, URIRef(f"{SCHEMA}{predicate}"), child)) + graph.add((child, URIRef(f"{SCHEMA}description"), Literal(key))) + generator = CanonicalIdGenerator(strategy=strategy) + generator.apply(graph, DATASET, language="ja") + assert len(set(graph.subjects(RDF.type, URIRef(f"{SCHEMA}{type_name}")))) == len( + names + ) + for key in ("a", "b", "c")[: len(names)]: + assert ( + len(set(graph.subjects(URIRef(f"{SCHEMA}description"), Literal(key)))) == 1 + ) + first_pass = set(graph) + generator.apply(graph, DATASET, language="ja") + assert set(graph) == first_pass + + +def test_unicode_questions_linked_to_article_remain_idempotent() -> None: + graph = Graph() + article = _entity(graph, "article", "Article", "Article") + faq = _entity(graph, "faq", "FAQ", "FAQPage") + graph.add((article, URIRef(f"{SCHEMA}subjectOf"), faq)) + for key, name in (("a", "大阪"), ("b", "東京"), ("c", "大阪")): + child = _entity(graph, key, name, "Question") + graph.add((faq, URIRef(f"{SCHEMA}mainEntity"), child)) + graph.add((child, URIRef(f"{SCHEMA}description"), Literal(key))) + generator = CanonicalIdGenerator() + generator.apply(graph, DATASET, language="ja") + assert len(set(graph.subjects(RDF.type, URIRef(f"{SCHEMA}Question")))) == 3 + first_pass = set(graph) + generator.apply(graph, DATASET, language="ja") + assert set(graph) == first_pass diff --git a/tests/kg_build/postprocessors/processors/test_slug.py b/tests/kg_build/postprocessors/processors/test_slug.py new file mode 100644 index 0000000..6d3e84d --- /dev/null +++ b/tests/kg_build/postprocessors/processors/test_slug.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import hashlib +from concurrent.futures import ThreadPoolExecutor + +import pytest + +import wordlift_sdk.kg_build.postprocessors.processors.slug as slug_module +from wordlift_sdk.kg_build.postprocessors.processors.slug import ( + identity_slug, + normalize_slug, +) + + +@pytest.mark.parametrize( + ("value", "language", "expected"), + [ + (" Hello__World!! ", None, "hello-world"), + ("Müller Straße", "de-DE", "mueller-strasse"), + ("MÜLLER", " DE_de ", "mueller"), + ("Müller", "tr", "muller"), + ("ışık", "tr", "isik"), + ("Łódź smørrebrød", None, "lodz-smorrebrod"), + ("Москва", "ru", "moskva"), + ("Київ", "uk", "kyyiv"), + ("Αθήνα", "el", "athena"), + ("مرحبا", "ar", "mrhba"), + ("שלום", "he", "slwm"), + ("東京", "zh-Hant-TW", "dong-jing"), + ("東京", "cmn-Hans-CN", "dong-jing"), + ("東京", "ja", "thing"), + ("東京", "yue", "thing"), + ("東京", "zh-yue-Hant", "thing"), + ("東京", "zh-min-nan", "thing"), + ("東京", None, "thing"), + ("Москва", "unknown", "thing"), + ("ひらがな カタカナ ｶﾀｶﾅ", "ja", "hiragana-katakana-katakana"), + ("서울", "ko", "seoul"), + ("हिन्दी", "hi", "hindi"), + ("", None, "thing"), + ("! 🎉 !", None, "thing"), + ], +) +def test_readable_transliteration(value, language, expected) -> None: + assert normalize_slug(value, language) == expected + + +def test_identity_keeps_original_non_ascii_distinctions() -> None: + digest = hashlib.sha256("müller".encode("utf-8")).hexdigest() + assert identity_slug("Müller", "de") == f"mueller-{digest}" + assert identity_slug("Mueller", "de") == "mueller" + assert identity_slug("Müller", "de", has_url=True) == "mueller" + assert identity_slug("東京", "ja") != identity_slug("大阪", "ja") + assert identity_slug("Tokyo 東京", "ja") != identity_slug("Tokyo 大阪", "ja") + assert identity_slug("Tokyo 東京", "ja").startswith("tokyo-") + + +def test_equivalent_unicode_case_and_whitespace_have_identical_ids() -> None: + assert identity_slug(" MÜLLER ", "de") == identity_slug("Mu\u0308ller", "de") + assert normalize_slug("Mu\u0308ller", "de") == "mueller" + + +def test_ascii_compatibility_and_empty_fallback() -> None: + assert identity_slug(" Hello__World!! ") == "hello-world" + assert identity_slug("!!!") == "thing" + assert identity_slug("") == "thing" + assert identity_slug("🎉").startswith("thing-") + + +def test_concurrent_languages_do_not_share_transform_state() -> None: + inputs = [("Müller", "de"), ("Müller", "tr"), ("東京", "ja"), ("東京", "zh")] * 8 + with ThreadPoolExecutor(max_workers=4) as executor: + results = list(executor.map(lambda args: normalize_slug(*args), inputs)) + assert results == ["mueller", "muller", "thing", "dong-jing"] * 8 + + +@pytest.mark.parametrize("language", ["de", "tr", "ja"]) +def test_unsupported_icu_version_fails_before_transliteration(monkeypatch, language): + monkeypatch.setattr(slug_module, "ICU_VERSION", "78.0") + with pytest.raises(RuntimeError, match="Install ICU 74.2 and rebuild PyICU"): + normalize_slug("Müller", language) + assert normalize_slug("Hello World", language) == "hello-world" + + +def test_surrounding_unicode_whitespace_does_not_change_identity() -> None: + assert identity_slug("\u00a0hello") == identity_slug("hello") diff --git a/tests/kg_build/postprocessors/test_postprocessors.py b/tests/kg_build/postprocessors/test_postprocessors.py index ed812ab..56ffbb6 100644 --- a/tests/kg_build/postprocessors/test_postprocessors.py +++ b/tests/kg_build/postprocessors/test_postprocessors.py @@ -133,6 +133,70 @@ def test_build_context_accepts_missing_account_key() -> None: assert context.profile["settings"]["api_url"] == "https://profile.example.com" +@pytest.mark.parametrize("runtime", ["oneshot", "persistent"]) +def test_account_language_reaches_allocator_and_generator_per_job( + tmp_path: Path, runtime: str +) -> None: + _write( + tmp_path / "language_pp.py", + """ + from rdflib import URIRef + from wordlift_sdk.kg_build.postprocessors.processors.id_postprocessor import ( + CanonicalIdsPostprocessor, + ) + + class LanguagePostprocessor: + def __init__(self): + self.canonical = CanonicalIdsPostprocessor() + + def process_graph(self, graph, context): + expected = context.ids.new_independent( + graph, type_name="Thing", base_value="Müller" + ) + result = self.canonical.process_graph(graph, context) + assert expected in set(result.subjects()) + return result + """, + ) + spec = PostprocessorSpec( + class_path="language_pp:LanguagePostprocessor", + python=sys.executable, + timeout_seconds=30, + enabled=True, + keep_temp_on_error=False, + ) + cls = ( + PersistentSubprocessPostprocessor + if runtime == "persistent" + else OneshotSubprocessPostprocessor + ) + processor = cls(spec=spec, root_dir=tmp_path) + try: + for language, prefix in [ + ("de-DE", "mueller-"), + ("tr", "muller-"), + (None, "muller-"), + ]: + context = _sample_context() + context.account.language = language + graph = Graph() + graph.add( + ( + URIRef("https://example.com/person"), + URIRef("http://schema.org/name"), + Literal("Müller"), + ) + ) + output = processor.process_graph(graph, context) + assert output is not None + assert len(set(output.subjects())) == 1 + assert str(next(output.subjects())).startswith( + f"https://data.example.com/things/{prefix}" + ) + finally: + processor.close() + + def test_manifest_precedence_prefers_selected_profile_file(tmp_path: Path) -> None: root = tmp_path _write( diff --git a/tests/kg_build/test_protocol.py b/tests/kg_build/test_protocol.py index 47f1369..cca06f3 100644 --- a/tests/kg_build/test_protocol.py +++ b/tests/kg_build/test_protocol.py @@ -811,6 +811,26 @@ def test_build_pp_context_exposes_resolved_profile_and_account_key() -> None: assert context.profile["settings"]["api_url"] == "https://profile-api.example.com" +@pytest.mark.parametrize( + "language,prefix", [("de", "mueller-"), ("tr", "muller-"), (None, "muller-")] +) +def test_build_pp_context_uses_account_language_for_ids(language, prefix) -> None: + host_context = _make_context() + host_context.account.language = language + protocol = ProfileImportProtocol( + context=host_context, profile=_make_profile(), root_dir=Path.cwd() + ) + context = protocol._build_pp_context( + "https://example.com/page", + WebPageScrapeResponse(web_page=WebPage(url="https://example.com/page")), + existing_web_page_id=None, + existing_import_hash=None, + ) + assert context.account is host_context.account + iri = context.ids.new_independent(Graph(), type_name="Thing", base_value="Müller") + assert str(iri).startswith(f"https://data.example.com/dataset/things/{prefix}") + + def test_build_pp_context_preserves_custom_profile_settings() -> None: protocol = ProfileImportProtocol( context=_make_context(), diff --git a/tests/tools/run_slice_smoke_imports.py b/tests/tools/run_slice_smoke_imports.py index f3b7505..edcffdd 100644 --- a/tests/tools/run_slice_smoke_imports.py +++ b/tests/tools/run_slice_smoke_imports.py @@ -172,6 +172,13 @@ def _smoke_ingestion() -> None: def _smoke_kg_build() -> None: kg_build = importlib.import_module("wordlift_sdk.kg_build") + icu = importlib.import_module("icu") + assert icu.ICU_VERSION == "74.2" + slug = importlib.import_module( + "wordlift_sdk.kg_build.postprocessors.processors.slug" + ) + assert slug.normalize_slug("Müller", "de") == "mueller" + assert slug.normalize_slug("Москва", "ru") == "moskva" with tempfile.TemporaryDirectory() as tmpdir: config_path = Path(tmpdir) / "worai.toml" config_path.write_text( @@ -188,7 +195,7 @@ def _smoke_kg_build() -> None: assert ( profile.resolve_mapping("https://example.com/article") == "default.yarrrml" ) - print("call ok: kg_build.load_profile_config") + print("call ok: kg_build.load_profile_config/language-aware slugs") SLICE_CALLS = { diff --git a/wordlift_sdk/kg_build/postprocessors/graph_io.py b/wordlift_sdk/kg_build/postprocessors/graph_io.py index 866189c..272b8bc 100644 --- a/wordlift_sdk/kg_build/postprocessors/graph_io.py +++ b/wordlift_sdk/kg_build/postprocessors/graph_io.py @@ -32,6 +32,7 @@ def _build_runner_payload(context: PostprocessorContext) -> dict[str, Any]: "url": context.url, "dataset_uri": dataset_uri, "country_code": country_code, + "language": getattr(account, "language", None), "account_key": account_key or None, "exports": context.exports, "existing_web_page_id": context.existing_web_page_id, diff --git a/wordlift_sdk/kg_build/postprocessors/oneshot.py b/wordlift_sdk/kg_build/postprocessors/oneshot.py index 6b8ceda..2ec1d3e 100644 --- a/wordlift_sdk/kg_build/postprocessors/oneshot.py +++ b/wordlift_sdk/kg_build/postprocessors/oneshot.py @@ -28,6 +28,7 @@ def _build_context(payload: dict[str, Any]) -> PostprocessorContext: account = SimpleNamespace( dataset_uri=dataset_uri, country_code=str(payload.get("country_code", "")).strip().lower(), + language=payload.get("language"), ) response_payload = payload.get("response", {}) or {} web_page_payload = response_payload.get("web_page", {}) or {} @@ -51,7 +52,9 @@ def _build_context(payload: dict[str, Any]) -> PostprocessorContext: if payload.get("existing_web_page_id") else None ), - ids=IdAllocator(dataset_uri) if dataset_uri else None, + ids=( + IdAllocator(dataset_uri, language=account.language) if dataset_uri else None + ), ) diff --git a/wordlift_sdk/kg_build/postprocessors/processors/id_allocator.py b/wordlift_sdk/kg_build/postprocessors/processors/id_allocator.py index de4e6ce..ed8cf42 100644 --- a/wordlift_sdk/kg_build/postprocessors/processors/id_allocator.py +++ b/wordlift_sdk/kg_build/postprocessors/processors/id_allocator.py @@ -1,30 +1,29 @@ from __future__ import annotations import hashlib -import re from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from rdflib import Graph, Literal, RDF, URIRef from ...id_policy import DEFAULT_ID_POLICY, IdPolicy +from .slug import identity_slug, normalize_slug as normalize_slug, source_digest SCHEMA = "http://schema.org/" -def normalize_slug(value: str) -> str: - lowered = value.strip().lower() - lowered = re.sub(r"[^\w\s-]", " ", lowered) - lowered = re.sub(r"[_\s]+", "-", lowered) - lowered = re.sub(r"-{2,}", "-", lowered).strip("-") - return lowered or "thing" - - class IdAllocator: """Allocate canonical IRIs for postprocessors using shared ID policy.""" - def __init__(self, dataset_uri: str, policy: IdPolicy | None = None) -> None: + def __init__( + self, + dataset_uri: str, + policy: IdPolicy | None = None, + *, + language: str | None = None, + ) -> None: self.dataset_uri = dataset_uri.rstrip("/") self.policy = policy or DEFAULT_ID_POLICY + self._language = language def assign( self, @@ -54,7 +53,9 @@ def assign( gtin = self._first_value(graph, subject, "gtin") if gtin: - new_iri = URIRef(f"{self.dataset_uri}/01/{normalize_slug(gtin)}") + new_iri = URIRef( + f"{self.dataset_uri}/01/{identity_slug(gtin, self._language)}" + ) if rewrite: self._swap_iri(graph, subject, new_iri) return new_iri @@ -242,11 +243,11 @@ def _entity_id( or self.policy.normalize_type_name(type_name) or "Thing" ) - base_slug = normalize_slug(raw_base) resolved_url = url_value or self._first_value(graph, subject, "url") + base_slug = identity_slug(raw_base, self._language, has_url=bool(resolved_url)) if resolved_url: return f"{base_slug}-{self._url_hash(resolved_url)}" - if force_index and index is not None: + if force_index and index is not None and source_digest(raw_base) is None: return f"{base_slug}-{index}" candidate = base_slug diff --git a/wordlift_sdk/kg_build/postprocessors/processors/id_generator.py b/wordlift_sdk/kg_build/postprocessors/processors/id_generator.py index d063f59..e344f48 100644 --- a/wordlift_sdk/kg_build/postprocessors/processors/id_generator.py +++ b/wordlift_sdk/kg_build/postprocessors/processors/id_generator.py @@ -9,18 +9,11 @@ from ...id_policy import DEFAULT_ID_POLICY, IdPolicy from ...iri_lookup import IriLookup +from .slug import identity_slug, normalize_slug as normalize_slug, source_digest SCHEMA = "http://schema.org/" -def normalize_slug(value: str) -> str: - lowered = value.strip().lower() - lowered = re.sub(r"[^\w\s-]", " ", lowered) - lowered = re.sub(r"[_\s]+", "-", lowered) - lowered = re.sub(r"-{2,}", "-", lowered).strip("-") - return lowered or "thing" - - class CanonicalIdGenerator: """Pure graph-ID canonicalization logic.""" @@ -37,13 +30,17 @@ def apply( graph: Graph, dataset_uri: str, iri_lookup: IriLookup | None = None, + *, + language: str | None = None, ) -> Graph: dataset_uri = dataset_uri.rstrip("/") if not dataset_uri: return graph if self._strategy == "dependency_graph": - return self._apply_dependency_graph_strategy(graph, dataset_uri, iri_lookup) + return self._apply_dependency_graph_strategy( + graph, dataset_uri, iri_lookup, language=language + ) root_subjects = self._root_subjects(graph) # Lookup-mapped IRIs are treated as authoritative and must not be @@ -57,6 +54,7 @@ def apply( root_subjects, locked_subjects, rewritten_subjects, + language=language, ) self._rewrite_entity_roots( graph, @@ -65,6 +63,7 @@ def apply( root_subjects, locked_subjects, rewritten_subjects, + language=language, ) self._rewrite_remaining_subjects( graph, @@ -73,8 +72,9 @@ def apply( root_subjects, locked_subjects, rewritten_subjects, + language=language, ) - self._rewrite_actions_as_dependents(graph) + self._rewrite_actions_as_dependents(graph, language=language) return graph # ------------------------------------------------------------------ @@ -86,6 +86,8 @@ def _apply_dependency_graph_strategy( graph: Graph, dataset_uri: str, iri_lookup: IriLookup | None, + *, + language: str | None = None, ) -> Graph: """Rewrite root IRIs first, then reparent dependents generically via IdPolicy.dependent_rules — no hard-coded root-type branches.""" @@ -110,6 +112,7 @@ def _apply_dependency_graph_strategy( old_page, default_base="web-page", url_value=str(url) if isinstance(url, (Literal, URIRef)) else None, + language=language, ) new_page = URIRef( f"{dataset_uri}" @@ -133,7 +136,9 @@ def _apply_dependency_graph_strategy( else: gtin = self._first_value(graph, entity, "gtin") if gtin: - entity_iri = URIRef(f"{dataset_uri}/01/{normalize_slug(gtin)}") + entity_iri = URIRef( + f"{dataset_uri}/01/{identity_slug(gtin, language)}" + ) else: entity_url = self._first_value(graph, entity, "url") subject_type = self._preferred_type_name(graph, entity) @@ -142,6 +147,7 @@ def _apply_dependency_graph_strategy( entity, default_base=subject_type, url_value=entity_url, + language=language, ) seen[entity_slug] += 1 if not entity_url and seen[entity_slug] > 1: @@ -174,7 +180,9 @@ def _apply_dependency_graph_strategy( else: gtin = self._first_value(graph, subject, "gtin") if gtin: - candidate = URIRef(f"{dataset_uri}/01/{normalize_slug(gtin)}") + candidate = URIRef( + f"{dataset_uri}/01/{identity_slug(gtin, language)}" + ) else: preferred_type = self._preferred_type_name(graph, subject) normalized_type = self._policy.normalize_type_name( @@ -186,6 +194,7 @@ def _apply_dependency_graph_strategy( subject, default_base=normalized_type, url_value=self._first_value(graph, subject, "url"), + language=language, ) candidate = URIRef(f"{dataset_uri}/{container}/{slug}") @@ -197,7 +206,7 @@ def _apply_dependency_graph_strategy( self._record_rewrite(rewritten_subjects, subject, new_iri) # Actions are still reparented as in legacy mode - self._rewrite_actions_as_dependents(graph) + self._rewrite_actions_as_dependents(graph, language=language) # Phase 4: generic dependent reparenting driven entirely by IdPolicy rules visited: set[URIRef] = set() @@ -209,7 +218,7 @@ def _apply_dependency_graph_strategy( }, key=str, ): - self._rewrite_dependents_by_policy(graph, iri, visited) + self._rewrite_dependents_by_policy(graph, iri, visited, language=language) return graph @@ -218,6 +227,8 @@ def _rewrite_dependents_by_policy( graph: Graph, parent_iri: URIRef, visited: set[URIRef], + *, + language: str | None = None, ) -> None: """Recursively reparent dependent nodes under *parent_iri* by walking every rule in IdPolicy.dependent_rules whose parent_predicates lead to @@ -246,12 +257,14 @@ def _rewrite_dependents_by_policy( count = len(children) for idx, child in enumerate(children, start=1): slug = self._dependent_slug( - graph, child, rule.child_type, idx, count + graph, child, rule.child_type, idx, count, language=language ) new_child = URIRef(f"{parent_iri}/{container}/{slug}") new_child = self._ensure_unique_subject_iri(graph, child, new_child) self._swap_iri(graph, child, new_child) - self._rewrite_dependents_by_policy(graph, new_child, visited) + self._rewrite_dependents_by_policy( + graph, new_child, visited, language=language + ) def _dependent_slug( self, @@ -260,6 +273,8 @@ def _dependent_slug( child_type: str, index: int, count: int, + *, + language: str | None = None, ) -> str: """Slug for a policy-dependent child node. @@ -273,10 +288,10 @@ def _dependent_slug( if isinstance(value, (Literal, URIRef)): text = str(value).strip() if text: - base = normalize_slug(text) or "thing" + base = identity_slug(text, language, has_url=bool(url_value)) if url_value: return f"{base}-{self._url_hash(url_value)}" - if count > 1: + if count > 1 and source_digest(text) is None: return f"{base}-{index}" return base @@ -301,6 +316,8 @@ def _rewrite_remaining_subjects( root_subjects: set[URIRef], locked_subjects: set[URIRef], rewritten_subjects: dict[URIRef, URIRef], + *, + language: str | None = None, ) -> None: subjects = sorted( {s for s in graph.subjects() if isinstance(s, URIRef)}, key=str @@ -328,7 +345,9 @@ def _rewrite_remaining_subjects( else: gtin = self._first_value(graph, subject, "gtin") if gtin: - candidate = URIRef(f"{dataset_uri}/01/{normalize_slug(gtin)}") + candidate = URIRef( + f"{dataset_uri}/01/{identity_slug(gtin, language)}" + ) else: preferred_type = self._preferred_type_name(graph, subject) normalized_type = self._policy.normalize_type_name( @@ -340,6 +359,7 @@ def _rewrite_remaining_subjects( subject, default_base=normalized_type, url_value=self._first_value(graph, subject, "url"), + language=language, ) candidate = URIRef(f"{dataset_uri}/{container}/{slug}") @@ -352,10 +372,12 @@ def _rewrite_remaining_subjects( # Rewrite FAQPage/Question/Answer nodes linked via subjectOf and # Rating nodes linked via reviewRating as nested dependents. self._rewrite_entity_linked_faq_and_rating( - graph, new_iri, already_processed + graph, new_iri, already_processed, language=language ) - def _rewrite_actions_as_dependents(self, graph: Graph) -> None: + def _rewrite_actions_as_dependents( + self, graph: Graph, *, language: str | None = None + ) -> None: action_type = URIRef(f"{SCHEMA}Action") actions = sorted( { @@ -380,6 +402,7 @@ def _rewrite_actions_as_dependents(self, graph: Graph) -> None: action, default_base="action", url_value=self._first_value(graph, action, "url"), + language=language, ) candidate = URIRef(f"{prefix}{slug}") new_iri = self._ensure_unique_subject_iri(graph, action, candidate) @@ -457,6 +480,8 @@ def _rewrite_pages_and_children( root_subjects: set[URIRef], locked_subjects: set[URIRef], rewritten_subjects: dict[URIRef, URIRef], + *, + language: str | None = None, ) -> None: page_nodes = { subject @@ -477,18 +502,21 @@ def _rewrite_pages_and_children( old_page, default_base="web-page", url_value=str(url) if isinstance(url, (Literal, URIRef)) else None, + language=language, ) new_page = URIRef( f"{dataset_uri}/{self._policy.container_for_type('WebPage')}/{page_slug}" ) self._swap_iri(graph, old_page, new_page) self._record_rewrite(rewritten_subjects, old_page, new_page) - self._rewrite_faq(graph, new_page) - self._rewrite_videos(graph, new_page) - self._rewrite_images(graph, new_page) + self._rewrite_faq(graph, new_page, language=language) + self._rewrite_videos(graph, new_page, language=language) + self._rewrite_images(graph, new_page, language=language) self._rewrite_howto(graph, new_page) - def _rewrite_faq(self, graph: Graph, page_iri: URIRef) -> None: + def _rewrite_faq( + self, graph: Graph, page_iri: URIRef, *, language: str | None = None + ) -> None: faq_nodes: set[URIRef] = set() for obj in graph.objects(page_iri, URIRef(f"{SCHEMA}hasPart")): if isinstance(obj, URIRef) and self._is_typed_as(graph, obj, "FAQPage"): @@ -515,9 +543,13 @@ def _rewrite_faq(self, graph: Graph, page_iri: URIRef) -> None: url_value=None, index=q_idx, force_index=len(questions) > 1, + language=language, ) q_container = self._policy.container_for_type("Question") new_question = URIRef(f"{new_faq}/{q_container}/{question_slug}") + new_question = self._ensure_unique_subject_iri( + graph, question, new_question + ) self._swap_iri(graph, question, new_question) answer = graph.value(new_question, URIRef(f"{SCHEMA}acceptedAnswer")) @@ -531,6 +563,8 @@ def _rewrite_entity_linked_faq_and_rating( graph: Graph, entity_iri: URIRef, already_processed: set[URIRef], + *, + language: str | None = None, ) -> None: """Rewrite FAQPage/Question/Answer and Rating nodes that are dependents of a non-page entity (e.g. Review, Article) via: @@ -572,9 +606,13 @@ def _rewrite_entity_linked_faq_and_rating( url_value=None, index=q_idx, force_index=len(questions) > 1, + language=language, ) q_container = self._policy.container_for_type("Question") new_question = URIRef(f"{new_faq}/{q_container}/{question_slug}") + new_question = self._ensure_unique_subject_iri( + graph, question, new_question + ) self._swap_iri(graph, question, new_question) already_processed.add(original_q) @@ -602,7 +640,9 @@ def _rewrite_entity_linked_faq_and_rating( self._swap_iri(graph, rating, new_rating) already_processed.add(original_r) - def _rewrite_videos(self, graph: Graph, page_iri: URIRef) -> None: + def _rewrite_videos( + self, graph: Graph, page_iri: URIRef, *, language: str | None = None + ) -> None: videos: set[URIRef] = set() for obj in graph.objects(page_iri, URIRef(f"{SCHEMA}video")): if isinstance(obj, URIRef) and ( @@ -622,12 +662,16 @@ def _rewrite_videos(self, graph: Graph, page_iri: URIRef) -> None: url_value=self._first_value(graph, video, "embedUrl", "contentUrl"), index=idx, force_index=len(videos) > 1, + language=language, ) video_container = self._policy.container_for_type("VideoObject") new_video = URIRef(f"{page_iri}/{video_container}/{video_slug}") + new_video = self._ensure_unique_subject_iri(graph, video, new_video) self._swap_iri(graph, video, new_video) - def _rewrite_images(self, graph: Graph, page_iri: URIRef) -> None: + def _rewrite_images( + self, graph: Graph, page_iri: URIRef, *, language: str | None = None + ) -> None: images: set[URIRef] = set() for obj in graph.objects(page_iri, URIRef(f"{SCHEMA}image")): if isinstance(obj, URIRef) and ( @@ -647,9 +691,11 @@ def _rewrite_images(self, graph: Graph, page_iri: URIRef) -> None: url_value=self._first_value(graph, image, "contentUrl"), index=idx, force_index=len(images) > 1, + language=language, ) image_container = self._policy.container_for_type("ImageObject") new_image = URIRef(f"{page_iri}/{image_container}/{image_slug}") + new_image = self._ensure_unique_subject_iri(graph, image, new_image) self._swap_iri(graph, image, new_image) def _rewrite_howto(self, graph: Graph, page_iri: URIRef) -> None: @@ -687,6 +733,8 @@ def _rewrite_entity_roots( root_subjects: set[URIRef], locked_subjects: set[URIRef], rewritten_subjects: dict[URIRef, URIRef], + *, + language: str | None = None, ) -> None: products = { subject @@ -703,7 +751,9 @@ def _rewrite_entity_roots( else: gtin = self._first_value(graph, product, "gtin") if gtin: - product_iri = URIRef(f"{dataset_uri}/01/{normalize_slug(gtin)}") + product_iri = URIRef( + f"{dataset_uri}/01/{identity_slug(gtin, language)}" + ) else: product_url = self._first_value(graph, product, "url") subject_type = self._preferred_type_name(graph, product) @@ -712,6 +762,7 @@ def _rewrite_entity_roots( product, default_base=subject_type, url_value=product_url, + language=language, ) seen[product_slug] += 1 if not product_url and seen[product_slug] > 1: @@ -781,12 +832,13 @@ def _entity_slug( url_value: str | None, index: int | None = None, force_index: bool = False, + language: str | None = None, ) -> str: base = self._base_from_priority(graph, subject) or default_base - slug = normalize_slug(base) or "thing" + slug = identity_slug(base, language, has_url=bool(url_value)) if url_value: return f"{slug}-{self._url_hash(url_value)}" - if force_index and index is not None: + if force_index and index is not None and source_digest(base) is None: return f"{slug}-{index}" return slug diff --git a/wordlift_sdk/kg_build/postprocessors/processors/id_postprocessor.py b/wordlift_sdk/kg_build/postprocessors/processors/id_postprocessor.py index ae51a92..09467d8 100644 --- a/wordlift_sdk/kg_build/postprocessors/processors/id_postprocessor.py +++ b/wordlift_sdk/kg_build/postprocessors/processors/id_postprocessor.py @@ -61,7 +61,12 @@ def process_graph(self, graph: Graph, context) -> Graph: if not dataset_uri: return graph iri_lookup = self._iri_lookup or self._lookup_from_context(context) - return self._generator.apply(graph, dataset_uri, iri_lookup=iri_lookup) + return self._generator.apply( + graph, + dataset_uri, + iri_lookup=iri_lookup, + language=getattr(context.account, "language", None), + ) def _lookup_from_context(self, context) -> IriLookup | None: extensions = getattr(context, "extensions", None) diff --git a/wordlift_sdk/kg_build/postprocessors/processors/slug.py b/wordlift_sdk/kg_build/postprocessors/processors/slug.py new file mode 100644 index 0000000..948bf24 --- /dev/null +++ b/wordlift_sdk/kg_build/postprocessors/processors/slug.py @@ -0,0 +1,86 @@ +"""Language-aware ASCII slugs and stable identity suffixes.""" + +from __future__ import annotations + +import hashlib +import re +import unicodedata + +from icu import ICU_VERSION, Transliterator + + +_TRANSFORMS = { + "ru": "Russian-Latin/BGN", + "uk": "Ukrainian-Latin/BGN", + "el": "Greek-Latin", + "ar": "Arabic-Latin", + "he": "Hebrew-Latin", + "ja": "Hiragana-Latin; Katakana-Latin", + "ko": "Hangul-Latin", + "hi": "Devanagari-Latin", +} +_GERMAN_REPLACEMENTS = str.maketrans( + {"ä": "ae", "ö": "oe", "ü": "ue", "Ä": "Ae", "Ö": "Oe", "Ü": "Ue"} +) + + +def _language_parts(language: str | None) -> list[str]: + return (language or "").strip().lower().replace("_", "-").split("-") + + +def _transform(language: str | None) -> str: + parts = _language_parts(language) + primary = parts[0] + # Han readings are Mandarin only. Do not apply them to Cantonese (including + # zh-yue), other Chinese extlangs, Japanese kanji, or an unknown language. + mandarin = primary in {"zh", "cmn"} and all( + len(part) != 3 or not part.isalpha() or part == "cmn" for part in parts[1:] + ) + route = "Han-Latin" if mandarin else _TRANSFORMS.get(primary) + return f"{route}; Latin-ASCII" if route else "Latin-ASCII" + + +def normalize_slug(value: str, language: str | None = None) -> str: + """Return a readable ASCII slug; unsupported script text is omitted. + + Use ``identity_slug`` when the result identifies an entity: transliteration + is lossy, and unsupported text alone yields the readable fallback ``thing``. + """ + value = unicodedata.normalize("NFC", value) + if not value.isascii() and ICU_VERSION != "74.2": + raise RuntimeError( + "Canonical ID transliteration requires ICU 74.2 for stable output; " + f"found ICU {ICU_VERSION}. Install ICU 74.2 and rebuild PyICU against it." + ) + if _language_parts(language)[0] == "de": + value = value.translate(_GERMAN_REPLACEMENTS) + if not value.isascii(): + # ICU objects are mutable: each call owns its transform, including when + # ingestion workers normalize slugs concurrently. + transform = Transliterator.createInstance(_transform(language)) + value = transform.transliterate(value) + lowered = value.strip().lower() + lowered = re.sub(r"[^\w\s-]", " ", lowered, flags=re.ASCII) + lowered = re.sub(r"[_\s]+", "-", lowered) + lowered = re.sub(r"-{2,}", "-", lowered).strip("-") + return lowered or "thing" + + +def identity_slug( + value: str, language: str | None = None, *, has_url: bool = False +) -> str: + """Disambiguate non-ASCII input unless the caller supplies a URL hash.""" + slug = normalize_slug(value, language) + digest = source_digest(value) if not has_url else None + if digest is not None: + return f"{slug}-{digest}" + return slug + + +def source_digest(value: str) -> str | None: + """Stable identity for non-ASCII source text, independent of romanization.""" + normalized = unicodedata.normalize("NFC", value.strip()) + if normalized.isascii(): + return None + original = unicodedata.normalize("NFC", normalized.strip().lower()) + return hashlib.sha256(original.encode("utf-8")).hexdigest() diff --git a/wordlift_sdk/kg_build/protocol.py b/wordlift_sdk/kg_build/protocol.py index 94a8bf9..366cb29 100644 --- a/wordlift_sdk/kg_build/protocol.py +++ b/wordlift_sdk/kg_build/protocol.py @@ -500,7 +500,13 @@ def _build_pp_context( existing_import_hash: str | None, ) -> PostprocessorContext: dataset_uri = self._dataset_uri - ids = IdAllocator(dataset_uri) if dataset_uri else None + ids = ( + IdAllocator( + dataset_uri, language=getattr(self.context.account, "language", None) + ) + if dataset_uri + else None + ) profile_payload = asdict(self.profile) profile_settings = dict(profile_payload.get("settings", {}) or {}) profile_settings.setdefault("api_url", "https://api.wordlift.io") From 169d36148b2f5395cab639d32f46c9a13cce501a Mon Sep 17 00:00:00 2001 From: David Riccitelli Date: Tue, 8 Sep 2026 13:34:47 +0300 Subject: [PATCH 2/2] fix: preserve duplicate IDs and Mandarin language extensions --- .../processors/test_id_allocator.py | 25 +++++++++++++++++++ .../postprocessors/processors/test_slug.py | 2 ++ .../postprocessors/processors/id_allocator.py | 2 +- .../postprocessors/processors/slug.py | 11 +++++--- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/tests/kg_build/postprocessors/processors/test_id_allocator.py b/tests/kg_build/postprocessors/processors/test_id_allocator.py index 4eb85b7..0f7aec9 100644 --- a/tests/kg_build/postprocessors/processors/test_id_allocator.py +++ b/tests/kg_build/postprocessors/processors/test_id_allocator.py @@ -137,3 +137,28 @@ def test_multilingual_allocator_identity_and_url_hash(name, language, slug) -> N ) == child ) + + +@pytest.mark.parametrize("force_index", [False, True]) +def test_duplicate_unicode_assignments_reuse_their_existing_suffix(force_index): + allocator = IdAllocator("https://data.example.com/dataset", language="ja") + graph = Graph() + originals = [URIRef(f"https://example.com/{index}") for index in range(3)] + for index, subject in enumerate(originals): + graph.add((subject, URIRef("http://schema.org/name"), Literal("東京"))) + graph.add( + (subject, URIRef("http://schema.org/description"), Literal(str(index))) + ) + allocated = [ + allocator.assign(graph, subject, force_index=force_index, index=index) + for index, subject in enumerate(originals, start=1) + ] + assert len(set(allocated)) == 3 + before = set(graph) + for _ in range(2): + for index, subject in reversed(list(enumerate(allocated, start=1))): + assert ( + allocator.assign(graph, subject, force_index=force_index, index=index) + == subject + ) + assert set(graph) == before diff --git a/tests/kg_build/postprocessors/processors/test_slug.py b/tests/kg_build/postprocessors/processors/test_slug.py index 6d3e84d..b37c26d 100644 --- a/tests/kg_build/postprocessors/processors/test_slug.py +++ b/tests/kg_build/postprocessors/processors/test_slug.py @@ -27,6 +27,8 @@ ("مرحبا", "ar", "mrhba"), ("שלום", "he", "slwm"), ("東京", "zh-Hant-TW", "dong-jing"), + ("東京", "zh-Hant-x-foo", "dong-jing"), + ("東京", "zh-u-co-pinyin", "dong-jing"), ("東京", "cmn-Hans-CN", "dong-jing"), ("東京", "ja", "thing"), ("東京", "yue", "thing"), diff --git a/wordlift_sdk/kg_build/postprocessors/processors/id_allocator.py b/wordlift_sdk/kg_build/postprocessors/processors/id_allocator.py index ed8cf42..f52c238 100644 --- a/wordlift_sdk/kg_build/postprocessors/processors/id_allocator.py +++ b/wordlift_sdk/kg_build/postprocessors/processors/id_allocator.py @@ -255,7 +255,7 @@ def _entity_id( return candidate if any(str(s) == f"{path_prefix}{candidate}" for s in graph.subjects()): next_index = 2 - while any( + while URIRef(f"{path_prefix}{base_slug}-{next_index}") != subject and any( str(s) == f"{path_prefix}{base_slug}-{next_index}" for s in graph.subjects() ): diff --git a/wordlift_sdk/kg_build/postprocessors/processors/slug.py b/wordlift_sdk/kg_build/postprocessors/processors/slug.py index 948bf24..06a6e56 100644 --- a/wordlift_sdk/kg_build/postprocessors/processors/slug.py +++ b/wordlift_sdk/kg_build/postprocessors/processors/slug.py @@ -33,9 +33,14 @@ def _transform(language: str | None) -> str: primary = parts[0] # Han readings are Mandarin only. Do not apply them to Cantonese (including # zh-yue), other Chinese extlangs, Japanese kanji, or an unknown language. - mandarin = primary in {"zh", "cmn"} and all( - len(part) != 3 or not part.isalpha() or part == "cmn" for part in parts[1:] - ) + mandarin = primary in {"zh", "cmn"} + for part in parts[1:]: + # Extlangs precede script/region/extension/private-use subtags. + if len(part) != 3 or not part.isalpha(): + break + if part != "cmn": + mandarin = False + break route = "Han-Latin" if mandarin else _TRANSFORMS.get(primary) return f"{route}; Latin-ASCII" if route else "Latin-ASCII"