From 5be7acaff15cedc826963deca04b7f004d7b6bff Mon Sep 17 00:00:00 2001 From: JuiHsuanLee0303 Date: Tue, 28 Jul 2026 14:51:10 +0800 Subject: [PATCH 1/2] Reject unimplemented BPE options instead of ignoring them silently BPE and BpeTrainer accepted a set of HuggingFace-compatible options, stored them, and in the model's case serialized them back out, without ever acting on any of them. A caller who set one got wrong tokenization with no signal, and a round-tripped tokenizer.json looked unchanged, so nothing appeared wrong. Options that would change tokenization now raise UnsupportedFeatureError: dropout in (0, 1], and non-empty continuing_subword_prefix / end_of_word_suffix. A dropout outside [0, 1] raises ModelError instead, since that is invalid rather than unimplemented, matching HuggingFace's own rejection at construction. Default values stay silent. None, 0.0 and "" are genuine no-ops in HuggingFace too, so files that tokenize correctly today keep loading -- GPT-2-lineage exports ship "" for both affix options. BpeTrainer gets the same treatment for its two affix options, which it also stored and never read. Fixing only the model would have left half of the same defect in place. cache_capacity was being discarded entirely, without even an attribute assignment. It is now retained, but not rejected: a cache is a pure optimization, so its absence cannot change output. HuggingFace does not serialize it into tokenizer.json, so to_dict() stays faithful by omitting it. The rejection rules live in a shared module because they must stay identical in the model and the trainer. Co-Authored-By: Claude Opus 5 (1M context) --- docs/components.md | 21 +++++ src/ubi_tokenizer/_validation.py | 54 +++++++++++++ src/ubi_tokenizer/errors.py | 9 +++ src/ubi_tokenizer/models/bpe.py | 11 +++ src/ubi_tokenizer/trainers/bpe.py | 2 + tests/test_unsupported_flags.py | 123 ++++++++++++++++++++++++++++++ 6 files changed, 220 insertions(+) create mode 100644 src/ubi_tokenizer/_validation.py create mode 100644 tests/test_unsupported_flags.py diff --git a/docs/components.md b/docs/components.md index 77c10a9..5583e48 100644 --- a/docs/components.md +++ b/docs/components.md @@ -61,3 +61,24 @@ Reconstructs text from tokens. - `BpeTrainer(vocab_size, min_frequency, special_tokens=None, ...)` — in-memory greedy trainer. For large/distributed training use UbiTok; this is for small corpora and tests. + +## Unimplemented options + +`BPE` and `BpeTrainer` accept the full HuggingFace option set so that a +`tokenizer.json` round-trips unchanged. Options that are not implemented raise +`UnsupportedFeatureError` when set to a value that would change tokenization, +rather than being accepted and ignored: + +| Option | Accepted values | Rejected values | +| --- | --- | --- | +| `dropout` | `None`, `0.0` | anything in `(0, 1]` — outside `[0, 1]` raises `ModelError` | +| `continuing_subword_prefix` | `None`, `""` | any non-empty string | +| `end_of_word_suffix` | `None`, `""` | any non-empty string | + +The accepted values are genuine no-ops in HuggingFace as well, so files that +tokenize correctly today keep loading — GPT-2-lineage exports ship `""` for both +affix options. + +`cache_capacity` is accepted and retained but not yet used. It is not rejected +because a cache is a pure optimization: its absence cannot change output. +HuggingFace does not serialize it into `tokenizer.json`, so `to_dict()` omits it. diff --git a/src/ubi_tokenizer/_validation.py b/src/ubi_tokenizer/_validation.py new file mode 100644 index 0000000..7ab5629 --- /dev/null +++ b/src/ubi_tokenizer/_validation.py @@ -0,0 +1,54 @@ +"""Shared rejection rules for recognized-but-unimplemented options. + +The BPE model and the BPE trainer both accept options that they store, and in +the model's case serialize, without ever acting on them. Silently accepting +such an option hands the caller wrong tokenization with no signal, so a value +that would change behaviour is rejected instead. + +The rule has to stay identical in both places, so it lives here rather than +being duplicated. +""" + +from __future__ import annotations + +from typing import Optional + +from ubi_tokenizer.errors import ModelError, UnsupportedFeatureError + +_HINT = ( + "recognized but not implemented by ubi_tokenizer. Drop the option, or use " + "huggingface/tokenizers if you need it." +) + + +def reject_unimplemented_affixes( + continuing_subword_prefix: Optional[str], + end_of_word_suffix: Optional[str], +) -> None: + """Reject affix options that would change tokenization. + + ``None`` and ``""`` are genuine no-ops in HuggingFace and stay accepted -- + GPT-2-lineage ``tokenizer.json`` files ship ``""`` for both, and rejecting + those would break loading files that tokenize correctly today. + """ + for name, value in ( + ("continuing_subword_prefix", continuing_subword_prefix), + ("end_of_word_suffix", end_of_word_suffix), + ): + if value: + raise UnsupportedFeatureError(f"{name}={value!r} is {_HINT}") + + +def reject_unimplemented_dropout(dropout: Optional[float]) -> None: + """Reject BPE dropout. + + ``None`` and ``0.0`` both mean "no dropout" and stay accepted. Values + outside ``[0, 1]`` are invalid rather than unimplemented, which is why they + raise :class:`ModelError`; HuggingFace rejects them at construction too. + """ + if dropout is None: + return + if not 0.0 <= dropout <= 1.0: + raise ModelError(f"dropout={dropout!r} must be between 0 and 1, inclusive") + if dropout > 0.0: + raise UnsupportedFeatureError(f"dropout={dropout!r} is {_HINT}") diff --git a/src/ubi_tokenizer/errors.py b/src/ubi_tokenizer/errors.py index b72daaf..6907d75 100644 --- a/src/ubi_tokenizer/errors.py +++ b/src/ubi_tokenizer/errors.py @@ -13,3 +13,12 @@ class DeserializationError(UbiTokenizerError): class ModelError(UbiTokenizerError): """Raised for invalid model state (missing tokens, bad merges, etc.).""" + + +class UnsupportedFeatureError(UbiTokenizerError): + """Raised when a requested option is recognized but not implemented. + + Distinct from :class:`ModelError`: the request is well-formed, it simply + cannot be honoured. Raised eagerly so the caller finds out when the + tokenizer is built rather than getting silently wrong tokenization. + """ diff --git a/src/ubi_tokenizer/models/bpe.py b/src/ubi_tokenizer/models/bpe.py index 4be8077..16b1a86 100644 --- a/src/ubi_tokenizer/models/bpe.py +++ b/src/ubi_tokenizer/models/bpe.py @@ -10,6 +10,11 @@ from typing import Any, Optional +from ubi_tokenizer._validation import ( + reject_unimplemented_affixes, + reject_unimplemented_dropout, +) + class Model: """Abstract base for tokenization models.""" @@ -47,8 +52,14 @@ def __init__( byte_fallback: bool = False, ignore_merges: bool = False, ) -> None: + reject_unimplemented_dropout(dropout) + reject_unimplemented_affixes(continuing_subword_prefix, end_of_word_suffix) self.vocab: dict[str, int] = dict(vocab) if vocab else {} self.merges: list[tuple[str, str]] = [tuple(m) for m in (merges or [])] + # Accepted rather than rejected: a cache is a pure optimization, so its + # absence cannot change output. HuggingFace does not serialize it into + # tokenizer.json either, so to_dict() stays faithful by omitting it. + self.cache_capacity = cache_capacity self.dropout = dropout self.unk_token = unk_token self.continuing_subword_prefix = continuing_subword_prefix diff --git a/src/ubi_tokenizer/trainers/bpe.py b/src/ubi_tokenizer/trainers/bpe.py index 3fc3fa6..a021631 100644 --- a/src/ubi_tokenizer/trainers/bpe.py +++ b/src/ubi_tokenizer/trainers/bpe.py @@ -14,6 +14,7 @@ from collections.abc import Iterable, Iterator from typing import Optional +from ubi_tokenizer._validation import reject_unimplemented_affixes from ubi_tokenizer.byte_level import BYTE_ENCODER, encode_token_bytes from ubi_tokenizer.models.bpe import BPE from ubi_tokenizer.pre_tokenizers.byte_level import ByteLevel @@ -40,6 +41,7 @@ def __init__( continuing_subword_prefix: Optional[str] = None, end_of_word_suffix: Optional[str] = None, ) -> None: + reject_unimplemented_affixes(continuing_subword_prefix, end_of_word_suffix) self.vocab_size = vocab_size self.min_frequency = min_frequency self.special_tokens = list(special_tokens or []) diff --git a/tests/test_unsupported_flags.py b/tests/test_unsupported_flags.py new file mode 100644 index 0000000..89879d7 --- /dev/null +++ b/tests/test_unsupported_flags.py @@ -0,0 +1,123 @@ +"""Unimplemented BPE / trainer options must fail loudly instead of silently. + +``BPE`` and ``BpeTrainer`` accept several HuggingFace-compatible options that are +stored and serialized but never acted on. Accepting them silently means a caller +gets wrong tokenization with no signal, so a non-default value now raises. +Default values stay silent: they are true no-ops in HuggingFace too, and +GPT-2-lineage ``tokenizer.json`` files ship ``""`` for the affix options. +""" + +import pytest + +from ubi_tokenizer.errors import ModelError, UbiTokenizerError, UnsupportedFeatureError +from ubi_tokenizer.models.bpe import BPE +from ubi_tokenizer.trainers.bpe import BpeTrainer + + +def test_unsupported_feature_error_is_a_ubi_tokenizer_error(): + assert issubclass(UnsupportedFeatureError, UbiTokenizerError) + + +# -- dropout --------------------------------------------------------------- + + +@pytest.mark.parametrize("value", [None, 0.0]) +def test_no_op_dropout_is_accepted(value): + # HuggingFace treats both as "no dropout", so they must not raise. + assert BPE(dropout=value).dropout == value + + +@pytest.mark.parametrize("value", [0.1, 0.5, 1.0]) +def test_active_dropout_is_rejected(value): + with pytest.raises(UnsupportedFeatureError, match="dropout"): + BPE(dropout=value) + + +@pytest.mark.parametrize("value", [-0.1, 1.5]) +def test_out_of_range_dropout_is_rejected_as_invalid_model_state(value): + # HuggingFace rejects these at construction as invalid rather than + # unimplemented, so they are ModelError and not UnsupportedFeatureError. + with pytest.raises(ModelError, match="dropout"): + BPE(dropout=value) + + +# -- affix options -------------------------------------------------------- + + +@pytest.mark.parametrize("option", ["continuing_subword_prefix", "end_of_word_suffix"]) +@pytest.mark.parametrize("value", [None, ""]) +def test_no_op_affix_options_are_accepted(option, value): + assert getattr(BPE(**{option: value}), option) == value + + +@pytest.mark.parametrize( + ("option", "value"), + [("continuing_subword_prefix", "##"), ("end_of_word_suffix", "")], +) +def test_active_affix_options_are_rejected(option, value): + with pytest.raises(UnsupportedFeatureError, match=option): + BPE(**{option: value}) + + +@pytest.mark.parametrize( + ("option", "value"), + [("continuing_subword_prefix", "##"), ("end_of_word_suffix", "")], +) +def test_trainer_rejects_active_affix_options(option, value): + # The trainer stores these two and never reads them either; fixing only the + # model would leave half of the same defect in place. + with pytest.raises(UnsupportedFeatureError, match=option): + BpeTrainer(**{option: value}) + + +@pytest.mark.parametrize("option", ["continuing_subword_prefix", "end_of_word_suffix"]) +@pytest.mark.parametrize("value", [None, ""]) +def test_trainer_accepts_no_op_affix_options(option, value): + assert getattr(BpeTrainer(**{option: value}), option) == value + + +def test_trainer_defaults_are_accepted(): + trainer = BpeTrainer(vocab_size=64, min_frequency=1) + assert trainer.continuing_subword_prefix is None + assert trainer.end_of_word_suffix is None + + +# -- message quality ------------------------------------------------------- + + +def test_rejection_message_names_the_option_and_says_it_is_unimplemented(): + with pytest.raises(UnsupportedFeatureError) as excinfo: + BPE(end_of_word_suffix="") + message = str(excinfo.value) + assert "end_of_word_suffix" in message + assert "not implemented" in message.lower() + + +# -- cache_capacity is accepted, not rejected ------------------------------ + + +def test_cache_capacity_is_retained(): + # A cache is a pure optimization: its absence cannot change output, so this + # is accepted rather than rejected. It was previously discarded entirely. + assert BPE(cache_capacity=1234).cache_capacity == 1234 + + +def test_cache_capacity_stays_out_of_serialization(): + # HuggingFace does not serialize cache_capacity into tokenizer.json, so + # omitting it keeps to_dict() faithful rather than lossy. + assert "cache_capacity" not in BPE(cache_capacity=1234).to_dict() + + +# -- deserialization path -------------------------------------------------- + + +def test_unsupported_option_in_tokenizer_json_is_rejected_on_load(): + from ubi_tokenizer.tokenizer import Tokenizer + + payload = { + "version": "1.0", + "added_tokens": [], + "model": {"type": "BPE", "vocab": {"a": 0}, "merges": [], "dropout": 0.5}, + } + with pytest.raises(UnsupportedFeatureError, match="dropout"): + Tokenizer.from_dict(payload) From a7c6a7e6de7a23513a7ed6435371b651ec1e68a3 Mon Sep 17 00:00:00 2001 From: JuiHsuanLee0303 Date: Tue, 28 Jul 2026 14:55:35 +0800 Subject: [PATCH 2/2] Implement unk_token, fuse_unk, byte_fallback and ignore_merges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These four options were stored and serialized but never affected tokenization, so any model relying on them was silently mis-tokenized. Character resolution now runs before merging, which is the order HuggingFace uses and not an implementation detail: the tokens resolution produces take part in merges themselves. With <0xE4><0xB8> in the vocabulary and a merge over its two halves, 中 tokenizes as ['<0xE4><0xB8>', '<0xAD>']. tokenize() therefore splits into _resolve_symbols() followed by _apply_merges(), whose signature changes from a string to a sequence of already-resolved symbols. The merge loop body is unchanged. Behaviour, all measured against huggingface/tokenizers rather than derived from reading its source: - unk_token: out-of-vocabulary characters become the unk token. Without one they are dropped, which is what HuggingFace does; previously they reached Tokenizer._encode_single and became id -1. - fuse_unk: consecutive unresolvable characters collapse into a single unk. - byte_fallback: an out-of-vocabulary character becomes its UTF-8 bytes as <0xXX> tokens, uppercase and zero-padded. All-or-nothing per character: one missing byte token sends the whole character to unk instead of emitting a partial expansion. - ignore_merges: a piece already present in the vocabulary is emitted whole. Two behaviours are worth calling out. A pending unk is *not* flushed by a successful byte expansion, so with <0xE4> missing, '中é' gives ['<0xC3>', '<0xA9>', '']. That is probably an upstream quirk, but byte-parity with HuggingFace is what this package is for, so it is reproduced rather than corrected, and it is covered by fixtures. And byte_fallback is encode-only: the ByteLevel decoder renders <0xE4> as the ASCII characters of its own name, so round-tripping it needs a ByteFallback decoder that does not exist yet. Setting unk_token to a string absent from the vocabulary stays legal at construction and raises ModelError only when unresolvable input actually arrives, matching HuggingFace's laziness. Expectations come from tests/fixtures/hf_bpe_flags.json, recorded by scripts/gen_hf_flag_fixtures.py against a pinned tokenizers version. The test module reads that recording with the standard library only, so the suite still runs with no third-party package installed. The generator is the sole consumer of the new dev extra. Co-Authored-By: Claude Opus 5 (1M context) --- docs/components.md | 34 +- pyproject.toml | 4 + scripts/gen_hf_flag_fixtures.py | 239 +++++ src/ubi_tokenizer/models/bpe.py | 74 +- tests/fixtures/hf_bpe_flags.json | 1691 ++++++++++++++++++++++++++++++ tests/test_bpe_flags.py | 91 ++ 6 files changed, 2128 insertions(+), 5 deletions(-) create mode 100644 scripts/gen_hf_flag_fixtures.py create mode 100644 tests/fixtures/hf_bpe_flags.json create mode 100644 tests/test_bpe_flags.py diff --git a/docs/components.md b/docs/components.md index 5583e48..b6f009d 100644 --- a/docs/components.md +++ b/docs/components.md @@ -37,8 +37,38 @@ byte-mapped space. Turns one byte-mapped piece into subword tokens. - `BPE(vocab, merges, *, unk_token=None, ...)` — byte-pair encoding. `tokenize` - greedily applies the lowest-rank applicable merge until none remain (identical - to the UbiTok reference runtime). + resolves each character onto vocabulary tokens, then greedily applies the + lowest-rank applicable merge until none remain (identical to the UbiTok + reference runtime). + +Resolution happens **before** merging, because the tokens it produces take part +in merges themselves — a vocabulary containing `<0xE4><0xB8>` plus a merge over +its two halves tokenizes `中` as `['<0xE4><0xB8>', '<0xAD>']`. + +| Option | Effect | +| --- | --- | +| `unk_token` | out-of-vocabulary characters become this token. Without it they are **dropped**, matching HuggingFace | +| `fuse_unk` | consecutive unresolvable characters collapse into a single unk | +| `byte_fallback` | an out-of-vocabulary character becomes its UTF-8 bytes as `<0xXX>` tokens (uppercase, zero-padded). All-or-nothing per character: one missing byte token sends the whole character to unk | +| `ignore_merges` | a piece that is already a vocabulary entry is emitted whole, skipping merges. Required by GPT-4o / Llama-3 tokenizers | + +Setting `unk_token` to a string that is absent from the vocabulary is allowed at +construction and raises `ModelError` only if unresolvable input actually arrives, +which is when HuggingFace raises too. + +### Two behaviours worth knowing + +**A pending unk survives a byte-fallback hit.** HuggingFace defers at most one +unresolved character, and a successful byte expansion does not flush it. With +`<0xE4>` missing from the vocabulary, `'中é'` gives +`['<0xC3>', '<0xA9>', '']` — the unk for `中` lands *after* the bytes of `é`. +This is likely an upstream quirk, but byte-parity with HuggingFace is the point +of this package, so it is reproduced rather than corrected. + +**`byte_fallback` is encode-only.** The `ByteLevel` decoder maps each character +through the byte table, so it renders `<0xE4>` as the six ASCII characters of its +own name rather than as part of `中`. Round-tripping byte-fallback output needs a +`ByteFallback` decoder, which does not exist yet. ## PostProcessor (`ubi_tokenizer.processors`) diff --git a/pyproject.toml b/pyproject.toml index 4b8daf4..5e80d97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,10 @@ dependencies = [] [project.optional-dependencies] transcription = [] +# Only needed to regenerate tests/fixtures/hf_bpe_flags.json; the test suite +# itself runs on the standard library alone. The tokenizers pin is deliberate: +# the recorded ground truth includes version-sensitive behaviour. +dev = ["pytest", "tokenizers==0.23.1"] [project.scripts] ubi-tokenizer = "ubi_tokenizer.__main__:main" diff --git a/scripts/gen_hf_flag_fixtures.py b/scripts/gen_hf_flag_fixtures.py new file mode 100644 index 0000000..157a088 --- /dev/null +++ b/scripts/gen_hf_flag_fixtures.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Regenerate the HuggingFace ground-truth fixtures for BPE option behaviour. + +``tests/test_bpe_flags.py`` checks ``ubi_tokenizer``'s BPE against recorded +HuggingFace output. This script produces that recording, so the expectations are +measured rather than derived from reading the Rust source. + +It is the only part of the test setup that needs a third-party package, and it is +run by hand -- the test suite itself stays standard-library only, which is the +point of the whole project. Install the dev extra first: + + pip install -e ".[dev]" + python scripts/gen_hf_flag_fixtures.py + +The ``tokenizers`` version is pinned in the dev extra and recorded in the output. +Several behaviours captured here are quirky enough (see ``bf_partial_bytes``) +that regenerating against a different version could silently move ground truth. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +try: + import tokenizers + from tokenizers import models +except ImportError: # pragma: no cover - developer-facing script + raise SystemExit( + 'huggingface tokenizers is required to regenerate fixtures: pip install -e ".[dev]"' + ) + +OUTPUT = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / "hf_bpe_flags.json" + +BYTE_TOKENS = [f"<0x{value:02X}>" for value in range(256)] + + +def vocab_of(*groups: list[str]) -> dict[str, int]: + """Assign ids in listed order, skipping duplicates.""" + vocab: dict[str, int] = {} + for group in groups: + for token in group: + if token not in vocab: + vocab[token] = len(vocab) + return vocab + + +def bytes_except(*excluded: str) -> list[str]: + return [token for token in BYTE_TOKENS if token not in excluded] + + +# Inputs reused across the byte-fallback cases. "\t" (0x09), "\n" (0x0A) and +# "\x01" exercise bytes below 0x10, where an implementation that produced +# uppercase-but-not-zero-padded hex would otherwise pass unnoticed -- every +# CJK/accented character has bytes >= 0x80 and so is always two hex digits. +LOW_BYTE_INPUTS = ["\t", "\n", "\x01", "\x0f"] + +CASES: list[dict[str, Any]] = [ + { + "name": "baseline_no_flags", + "why": "merge loop must not regress", + "vocab": vocab_of(["a", "b", "c", "ab", "abc"]), + "merges": [("a", "b"), ("ab", "c")], + "flags": {}, + "inputs": ["abc", "ab", "a", "abz", "zz", "az", ""], + }, + { + "name": "ignore_merges_off_control", + "why": "control for the case below; proves the flag is what changes output", + "vocab": vocab_of(["x", "y", "xy", "z", "xz"]), + "merges": [("x", "z")], + "flags": {}, + "inputs": ["xy", "xz", "xyz", "xyxy", ""], + }, + { + "name": "ignore_merges_discriminating", + "why": "'xy' is in vocab but no merge reaches it, so the flag alone decides", + "vocab": vocab_of(["x", "y", "xy", "z", "xz"]), + "merges": [("x", "z")], + "flags": {"ignore_merges": True}, + "inputs": ["xy", "xz", "xyz", "xyxy", ""], + }, + { + "name": "drop_when_no_unk", + "why": "out-of-vocab characters are dropped, not turned into an invalid id", + "vocab": vocab_of(["a", "b"]), + "merges": [], + "flags": {}, + "inputs": ["zz", "azb", "az", "za", "中"], + }, + { + "name": "unk_only", + "why": "one unk per out-of-vocab character", + "vocab": vocab_of(["a", "b", "ab", ""]), + "merges": [("a", "b")], + "flags": {"unk_token": ""}, + "inputs": ["ab", "az", "zz", "az z", "中", ""], + }, + { + "name": "unk_fused", + "why": "consecutive pending unks coalesce", + "vocab": vocab_of(["a", "b", "ab", ""]), + "merges": [("a", "b")], + "flags": {"unk_token": "", "fuse_unk": True}, + "inputs": ["ab", "az", "zz", "az z", "中", "azza"], + }, + { + "name": "unk_token_present_in_vocab", + "why": "an in-vocab unk string must not swallow a following genuine unk", + "vocab": vocab_of(["a", "U"]), + "merges": [], + "flags": {"unk_token": "U", "fuse_unk": True}, + "inputs": ["Uz", "zU", "aUza", "UU"], + }, + { + "name": "byte_fallback_with_unk", + "why": "byte expansion, including bytes below 0x10", + "vocab": vocab_of(["a", ""], BYTE_TOKENS), + "merges": [], + "flags": {"unk_token": "", "byte_fallback": True}, + "inputs": ["a", "az", "中", "é", "abc", "€", *LOW_BYTE_INPUTS], + }, + { + "name": "byte_fallback_without_unk", + "why": "no unk to fall back on; misses are dropped", + "vocab": vocab_of(["a"], bytes_except("<0xE4>", "<0xB8>", "<0xAD>")), + "merges": [], + "flags": {"byte_fallback": True}, + "inputs": ["a", "az", "中", "é", "a中a", *LOW_BYTE_INPUTS], + }, + { + "name": "bare_ascii_absent_but_byte_token_present", + "why": "byte fallback also fires for plain ASCII when the bare char is missing", + "vocab": vocab_of([""], ["<0x61>", "<0x62>", "<0x63>"]), + "merges": [], + "flags": {"unk_token": "", "byte_fallback": True}, + "inputs": ["abc", "a", "abd"], + }, + { + "name": "bf_partial_bytes", + "why": "all-or-nothing per character, and the pending-unk slot is NOT flushed by a byte-fallback hit", + "vocab": vocab_of(["a", ""], bytes_except("<0xE4>")), + "merges": [], + "flags": {"unk_token": "", "byte_fallback": True}, + "inputs": ["中", "中é", "é中", "中中é", "中é中", "中中", "aé中", "中a"], + }, + { + "name": "bf_partial_bytes_fused", + "why": "same slot behaviour with coalescing on top", + "vocab": vocab_of(["a", ""], bytes_except("<0xE4>")), + "merges": [], + "flags": {"unk_token": "", "byte_fallback": True, "fuse_unk": True}, + "inputs": ["中", "中é", "é中", "中中é", "中é中", "中中", "aé中", "中a"], + }, + { + "name": "merge_over_byte_tokens", + "why": "byte-fallback tokens participate in merges, so resolution precedes merging", + "vocab": vocab_of( + ["a", "", "<0xE4>", "<0xB8>", "<0xAD>", "<0xE4><0xB8>", "a<0xE4>"] + ), + "merges": [("<0xE4>", "<0xB8>")], + "flags": {"unk_token": "", "byte_fallback": True}, + "inputs": ["中", "a中"], + }, + { + "name": "merge_over_byte_tokens_rank_order", + "why": "merge rank over byte tokens changes the result", + "vocab": vocab_of( + ["a", "", "<0xE4>", "<0xB8>", "<0xAD>", "<0xE4><0xB8>", "a<0xE4>"] + ), + "merges": [("a", "<0xE4>"), ("<0xE4>", "<0xB8>")], + "flags": {"unk_token": "", "byte_fallback": True}, + "inputs": ["中", "a中"], + }, + { + "name": "merge_over_unk_tokens", + "why": "generated unk tokens participate in merges too", + "vocab": vocab_of(["a", "", ""]), + "merges": [("", "")], + "flags": {"unk_token": ""}, + "inputs": ["zz", "azza", "z", "zzz"], + }, + { + "name": "fuse_unk_without_unk_token", + "why": "fuse_unk is inert with no unk token to fuse", + "vocab": vocab_of(["a", "b"]), + "merges": [], + "flags": {"fuse_unk": True}, + "inputs": ["azza", "zz", "ab"], + }, + { + "name": "ignore_merges_falls_through_to_unk", + "why": "a piece absent from vocab still takes the normal resolve path", + "vocab": vocab_of(["a", "b", "ab", ""]), + "merges": [("a", "b")], + "flags": {"ignore_merges": True, "unk_token": ""}, + "inputs": ["ab", "abz", "zz", "a"], + }, +] + + +def build(case: dict[str, Any]) -> dict[str, Any]: + model = models.BPE( + vocab=dict(case["vocab"]), + merges=[tuple(pair) for pair in case["merges"]], + **case["flags"], + ) + expected = {} + for text in case["inputs"]: + expected[text] = [token.value for token in model.tokenize(text)] + return { + "name": case["name"], + "why": case["why"], + "vocab": case["vocab"], + "merges": [list(pair) for pair in case["merges"]], + "flags": case["flags"], + "expected": expected, + } + + +def main() -> None: + payload = { + "_generated_by": "scripts/gen_hf_flag_fixtures.py", + "_ground_truth": f"huggingface tokenizers {tokenizers.__version__}", + "_warning": ( + "Do not hand-edit. Regenerate with the pinned tokenizers version; some " + "recorded behaviour is version-sensitive." + ), + "cases": [build(case) for case in CASES], + } + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + OUTPUT.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + total = sum(len(case["expected"]) for case in payload["cases"]) + print(f"wrote {OUTPUT.relative_to(Path.cwd())}: {len(payload['cases'])} cases, {total} inputs") + + +if __name__ == "__main__": + main() diff --git a/src/ubi_tokenizer/models/bpe.py b/src/ubi_tokenizer/models/bpe.py index 16b1a86..cf5280a 100644 --- a/src/ubi_tokenizer/models/bpe.py +++ b/src/ubi_tokenizer/models/bpe.py @@ -8,12 +8,14 @@ from __future__ import annotations +from collections.abc import Sequence from typing import Any, Optional from ubi_tokenizer._validation import ( reject_unimplemented_affixes, reject_unimplemented_dropout, ) +from ubi_tokenizer.errors import ModelError class Model: @@ -86,10 +88,76 @@ def set_vocab_and_merges( self.merges = [tuple(m) for m in merges] self._rebuild() - # -- core merge logic (lifted from ByteLevelBPE._apply_bpe) ------------ + # -- tokenization ------------------------------------------------------ def tokenize(self, piece: str) -> list[str]: - """Apply BPE merges to one byte-mapped pre-token piece.""" - symbols = tuple(piece) + """Turn one pre-token piece into subword tokens.""" + if not piece: + return [] + # A whole piece that is already a vocabulary entry is emitted as-is with + # no merging. GPT-4o / Llama-3 tokenizers rely on this. + if self.ignore_merges and piece in self.vocab: + return [piece] + return self._apply_merges(self._resolve_symbols(piece)) + + def _resolve_symbols(self, piece: str) -> list[str]: + """Map each character onto vocabulary tokens, resolving unknowns. + + Runs *before* merging, because the tokens produced here -- byte-fallback + tokens and generated unks alike -- take part in merges themselves. A + vocabulary containing ``<0xE4><0xB8>`` with a merge over its two halves + tokenizes ``中`` as ``['<0xE4><0xB8>', '<0xAD>']``. + + Unresolvable characters are dropped when there is no unk token, which is + what HuggingFace does. + """ + symbols: list[str] = [] + # HuggingFace holds at most one deferred unk, and what flushes it is + # load-bearing; see the byte-fallback branch below. + pending_unk = False + for character in piece: + if character in self.vocab: + if pending_unk: + symbols.append(self._unk_token_or_raise()) + pending_unk = False + symbols.append(character) + continue + if self.byte_fallback: + byte_tokens = [ + f"<0x{value:02X}>" for value in character.encode("utf-8") + ] + # All-or-nothing per character: a single missing byte token sends + # the whole character down the unk path rather than emitting a + # partial expansion. + if all(token in self.vocab for token in byte_tokens): + # Deliberately leaves pending_unk alone. HuggingFace defers a + # pending unk past a successful byte expansion, so with + # <0xE4> missing, '中é' gives ['<0xC3>', '<0xA9>', '']. + # Probably an upstream quirk, but byte-parity is the point of + # this package, so it is reproduced rather than corrected. + symbols.extend(byte_tokens) + continue + if self.unk_token is None: + continue + if pending_unk and not self.fuse_unk: + symbols.append(self._unk_token_or_raise()) + pending_unk = True + if pending_unk: + symbols.append(self._unk_token_or_raise()) + return symbols + + def _unk_token_or_raise(self) -> str: + """Return the unk token, or fail if the vocabulary has no id for it.""" + if self.unk_token not in self.vocab: + raise ModelError( + f"unk token {self.unk_token!r} is not in the vocabulary, so " + "out-of-vocabulary input cannot be represented" + ) + return self.unk_token + + # -- core merge logic (lifted from ByteLevelBPE._apply_bpe) ------------ + def _apply_merges(self, symbols: Sequence[str]) -> list[str]: + """Apply BPE merges to already-resolved symbols.""" + symbols = tuple(symbols) if len(symbols) < 2: return list(symbols) while True: diff --git a/tests/fixtures/hf_bpe_flags.json b/tests/fixtures/hf_bpe_flags.json new file mode 100644 index 0000000..678b13b --- /dev/null +++ b/tests/fixtures/hf_bpe_flags.json @@ -0,0 +1,1691 @@ +{ + "_generated_by": "scripts/gen_hf_flag_fixtures.py", + "_ground_truth": "huggingface tokenizers 0.23.1", + "_warning": "Do not hand-edit. Regenerate with the pinned tokenizers version; some recorded behaviour is version-sensitive.", + "cases": [ + { + "name": "baseline_no_flags", + "why": "merge loop must not regress", + "vocab": { + "a": 0, + "b": 1, + "c": 2, + "ab": 3, + "abc": 4 + }, + "merges": [ + [ + "a", + "b" + ], + [ + "ab", + "c" + ] + ], + "flags": {}, + "expected": { + "abc": [ + "abc" + ], + "ab": [ + "ab" + ], + "a": [ + "a" + ], + "abz": [ + "ab" + ], + "zz": [], + "az": [ + "a" + ], + "": [] + } + }, + { + "name": "ignore_merges_off_control", + "why": "control for the case below; proves the flag is what changes output", + "vocab": { + "x": 0, + "y": 1, + "xy": 2, + "z": 3, + "xz": 4 + }, + "merges": [ + [ + "x", + "z" + ] + ], + "flags": {}, + "expected": { + "xy": [ + "x", + "y" + ], + "xz": [ + "xz" + ], + "xyz": [ + "x", + "y", + "z" + ], + "xyxy": [ + "x", + "y", + "x", + "y" + ], + "": [] + } + }, + { + "name": "ignore_merges_discriminating", + "why": "'xy' is in vocab but no merge reaches it, so the flag alone decides", + "vocab": { + "x": 0, + "y": 1, + "xy": 2, + "z": 3, + "xz": 4 + }, + "merges": [ + [ + "x", + "z" + ] + ], + "flags": { + "ignore_merges": true + }, + "expected": { + "xy": [ + "xy" + ], + "xz": [ + "xz" + ], + "xyz": [ + "x", + "y", + "z" + ], + "xyxy": [ + "x", + "y", + "x", + "y" + ], + "": [] + } + }, + { + "name": "drop_when_no_unk", + "why": "out-of-vocab characters are dropped, not turned into an invalid id", + "vocab": { + "a": 0, + "b": 1 + }, + "merges": [], + "flags": {}, + "expected": { + "zz": [], + "azb": [ + "a", + "b" + ], + "az": [ + "a" + ], + "za": [ + "a" + ], + "中": [] + } + }, + { + "name": "unk_only", + "why": "one unk per out-of-vocab character", + "vocab": { + "a": 0, + "b": 1, + "ab": 2, + "": 3 + }, + "merges": [ + [ + "a", + "b" + ] + ], + "flags": { + "unk_token": "" + }, + "expected": { + "ab": [ + "ab" + ], + "az": [ + "a", + "" + ], + "zz": [ + "", + "" + ], + "az z": [ + "a", + "", + "", + "" + ], + "中": [ + "" + ], + "": [ + "", + "", + "", + "", + "" + ] + } + }, + { + "name": "unk_fused", + "why": "consecutive pending unks coalesce", + "vocab": { + "a": 0, + "b": 1, + "ab": 2, + "": 3 + }, + "merges": [ + [ + "a", + "b" + ] + ], + "flags": { + "unk_token": "", + "fuse_unk": true + }, + "expected": { + "ab": [ + "ab" + ], + "az": [ + "a", + "" + ], + "zz": [ + "" + ], + "az z": [ + "a", + "" + ], + "中": [ + "" + ], + "azza": [ + "a", + "", + "a" + ] + } + }, + { + "name": "unk_token_present_in_vocab", + "why": "an in-vocab unk string must not swallow a following genuine unk", + "vocab": { + "a": 0, + "U": 1 + }, + "merges": [], + "flags": { + "unk_token": "U", + "fuse_unk": true + }, + "expected": { + "Uz": [ + "U", + "U" + ], + "zU": [ + "U", + "U" + ], + "aUza": [ + "a", + "U", + "U", + "a" + ], + "UU": [ + "U", + "U" + ] + } + }, + { + "name": "byte_fallback_with_unk", + "why": "byte expansion, including bytes below 0x10", + "vocab": { + "a": 0, + "": 1, + "<0x00>": 2, + "<0x01>": 3, + "<0x02>": 4, + "<0x03>": 5, + "<0x04>": 6, + "<0x05>": 7, + "<0x06>": 8, + "<0x07>": 9, + "<0x08>": 10, + "<0x09>": 11, + "<0x0A>": 12, + "<0x0B>": 13, + "<0x0C>": 14, + "<0x0D>": 15, + "<0x0E>": 16, + "<0x0F>": 17, + "<0x10>": 18, + "<0x11>": 19, + "<0x12>": 20, + "<0x13>": 21, + "<0x14>": 22, + "<0x15>": 23, + "<0x16>": 24, + "<0x17>": 25, + "<0x18>": 26, + "<0x19>": 27, + "<0x1A>": 28, + "<0x1B>": 29, + "<0x1C>": 30, + "<0x1D>": 31, + "<0x1E>": 32, + "<0x1F>": 33, + "<0x20>": 34, + "<0x21>": 35, + "<0x22>": 36, + "<0x23>": 37, + "<0x24>": 38, + "<0x25>": 39, + "<0x26>": 40, + "<0x27>": 41, + "<0x28>": 42, + "<0x29>": 43, + "<0x2A>": 44, + "<0x2B>": 45, + "<0x2C>": 46, + "<0x2D>": 47, + "<0x2E>": 48, + "<0x2F>": 49, + "<0x30>": 50, + "<0x31>": 51, + "<0x32>": 52, + "<0x33>": 53, + "<0x34>": 54, + "<0x35>": 55, + "<0x36>": 56, + "<0x37>": 57, + "<0x38>": 58, + "<0x39>": 59, + "<0x3A>": 60, + "<0x3B>": 61, + "<0x3C>": 62, + "<0x3D>": 63, + "<0x3E>": 64, + "<0x3F>": 65, + "<0x40>": 66, + "<0x41>": 67, + "<0x42>": 68, + "<0x43>": 69, + "<0x44>": 70, + "<0x45>": 71, + "<0x46>": 72, + "<0x47>": 73, + "<0x48>": 74, + "<0x49>": 75, + "<0x4A>": 76, + "<0x4B>": 77, + "<0x4C>": 78, + "<0x4D>": 79, + "<0x4E>": 80, + "<0x4F>": 81, + "<0x50>": 82, + "<0x51>": 83, + "<0x52>": 84, + "<0x53>": 85, + "<0x54>": 86, + "<0x55>": 87, + "<0x56>": 88, + "<0x57>": 89, + "<0x58>": 90, + "<0x59>": 91, + "<0x5A>": 92, + "<0x5B>": 93, + "<0x5C>": 94, + "<0x5D>": 95, + "<0x5E>": 96, + "<0x5F>": 97, + "<0x60>": 98, + "<0x61>": 99, + "<0x62>": 100, + "<0x63>": 101, + "<0x64>": 102, + "<0x65>": 103, + "<0x66>": 104, + "<0x67>": 105, + "<0x68>": 106, + "<0x69>": 107, + "<0x6A>": 108, + "<0x6B>": 109, + "<0x6C>": 110, + "<0x6D>": 111, + "<0x6E>": 112, + "<0x6F>": 113, + "<0x70>": 114, + "<0x71>": 115, + "<0x72>": 116, + "<0x73>": 117, + "<0x74>": 118, + "<0x75>": 119, + "<0x76>": 120, + "<0x77>": 121, + "<0x78>": 122, + "<0x79>": 123, + "<0x7A>": 124, + "<0x7B>": 125, + "<0x7C>": 126, + "<0x7D>": 127, + "<0x7E>": 128, + "<0x7F>": 129, + "<0x80>": 130, + "<0x81>": 131, + "<0x82>": 132, + "<0x83>": 133, + "<0x84>": 134, + "<0x85>": 135, + "<0x86>": 136, + "<0x87>": 137, + "<0x88>": 138, + "<0x89>": 139, + "<0x8A>": 140, + "<0x8B>": 141, + "<0x8C>": 142, + "<0x8D>": 143, + "<0x8E>": 144, + "<0x8F>": 145, + "<0x90>": 146, + "<0x91>": 147, + "<0x92>": 148, + "<0x93>": 149, + "<0x94>": 150, + "<0x95>": 151, + "<0x96>": 152, + "<0x97>": 153, + "<0x98>": 154, + "<0x99>": 155, + "<0x9A>": 156, + "<0x9B>": 157, + "<0x9C>": 158, + "<0x9D>": 159, + "<0x9E>": 160, + "<0x9F>": 161, + "<0xA0>": 162, + "<0xA1>": 163, + "<0xA2>": 164, + "<0xA3>": 165, + "<0xA4>": 166, + "<0xA5>": 167, + "<0xA6>": 168, + "<0xA7>": 169, + "<0xA8>": 170, + "<0xA9>": 171, + "<0xAA>": 172, + "<0xAB>": 173, + "<0xAC>": 174, + "<0xAD>": 175, + "<0xAE>": 176, + "<0xAF>": 177, + "<0xB0>": 178, + "<0xB1>": 179, + "<0xB2>": 180, + "<0xB3>": 181, + "<0xB4>": 182, + "<0xB5>": 183, + "<0xB6>": 184, + "<0xB7>": 185, + "<0xB8>": 186, + "<0xB9>": 187, + "<0xBA>": 188, + "<0xBB>": 189, + "<0xBC>": 190, + "<0xBD>": 191, + "<0xBE>": 192, + "<0xBF>": 193, + "<0xC0>": 194, + "<0xC1>": 195, + "<0xC2>": 196, + "<0xC3>": 197, + "<0xC4>": 198, + "<0xC5>": 199, + "<0xC6>": 200, + "<0xC7>": 201, + "<0xC8>": 202, + "<0xC9>": 203, + "<0xCA>": 204, + "<0xCB>": 205, + "<0xCC>": 206, + "<0xCD>": 207, + "<0xCE>": 208, + "<0xCF>": 209, + "<0xD0>": 210, + "<0xD1>": 211, + "<0xD2>": 212, + "<0xD3>": 213, + "<0xD4>": 214, + "<0xD5>": 215, + "<0xD6>": 216, + "<0xD7>": 217, + "<0xD8>": 218, + "<0xD9>": 219, + "<0xDA>": 220, + "<0xDB>": 221, + "<0xDC>": 222, + "<0xDD>": 223, + "<0xDE>": 224, + "<0xDF>": 225, + "<0xE0>": 226, + "<0xE1>": 227, + "<0xE2>": 228, + "<0xE3>": 229, + "<0xE4>": 230, + "<0xE5>": 231, + "<0xE6>": 232, + "<0xE7>": 233, + "<0xE8>": 234, + "<0xE9>": 235, + "<0xEA>": 236, + "<0xEB>": 237, + "<0xEC>": 238, + "<0xED>": 239, + "<0xEE>": 240, + "<0xEF>": 241, + "<0xF0>": 242, + "<0xF1>": 243, + "<0xF2>": 244, + "<0xF3>": 245, + "<0xF4>": 246, + "<0xF5>": 247, + "<0xF6>": 248, + "<0xF7>": 249, + "<0xF8>": 250, + "<0xF9>": 251, + "<0xFA>": 252, + "<0xFB>": 253, + "<0xFC>": 254, + "<0xFD>": 255, + "<0xFE>": 256, + "<0xFF>": 257 + }, + "merges": [], + "flags": { + "unk_token": "", + "byte_fallback": true + }, + "expected": { + "a": [ + "a" + ], + "az": [ + "a", + "<0x7A>" + ], + "中": [ + "<0xE4>", + "<0xB8>", + "<0xAD>" + ], + "é": [ + "<0xC3>", + "<0xA9>" + ], + "abc": [ + "a", + "<0x62>", + "<0x63>" + ], + "€": [ + "<0xE2>", + "<0x82>", + "<0xAC>" + ], + "\t": [ + "<0x09>" + ], + "\n": [ + "<0x0A>" + ], + "\u0001": [ + "<0x01>" + ], + "\u000f": [ + "<0x0F>" + ] + } + }, + { + "name": "byte_fallback_without_unk", + "why": "no unk to fall back on; misses are dropped", + "vocab": { + "a": 0, + "<0x00>": 1, + "<0x01>": 2, + "<0x02>": 3, + "<0x03>": 4, + "<0x04>": 5, + "<0x05>": 6, + "<0x06>": 7, + "<0x07>": 8, + "<0x08>": 9, + "<0x09>": 10, + "<0x0A>": 11, + "<0x0B>": 12, + "<0x0C>": 13, + "<0x0D>": 14, + "<0x0E>": 15, + "<0x0F>": 16, + "<0x10>": 17, + "<0x11>": 18, + "<0x12>": 19, + "<0x13>": 20, + "<0x14>": 21, + "<0x15>": 22, + "<0x16>": 23, + "<0x17>": 24, + "<0x18>": 25, + "<0x19>": 26, + "<0x1A>": 27, + "<0x1B>": 28, + "<0x1C>": 29, + "<0x1D>": 30, + "<0x1E>": 31, + "<0x1F>": 32, + "<0x20>": 33, + "<0x21>": 34, + "<0x22>": 35, + "<0x23>": 36, + "<0x24>": 37, + "<0x25>": 38, + "<0x26>": 39, + "<0x27>": 40, + "<0x28>": 41, + "<0x29>": 42, + "<0x2A>": 43, + "<0x2B>": 44, + "<0x2C>": 45, + "<0x2D>": 46, + "<0x2E>": 47, + "<0x2F>": 48, + "<0x30>": 49, + "<0x31>": 50, + "<0x32>": 51, + "<0x33>": 52, + "<0x34>": 53, + "<0x35>": 54, + "<0x36>": 55, + "<0x37>": 56, + "<0x38>": 57, + "<0x39>": 58, + "<0x3A>": 59, + "<0x3B>": 60, + "<0x3C>": 61, + "<0x3D>": 62, + "<0x3E>": 63, + "<0x3F>": 64, + "<0x40>": 65, + "<0x41>": 66, + "<0x42>": 67, + "<0x43>": 68, + "<0x44>": 69, + "<0x45>": 70, + "<0x46>": 71, + "<0x47>": 72, + "<0x48>": 73, + "<0x49>": 74, + "<0x4A>": 75, + "<0x4B>": 76, + "<0x4C>": 77, + "<0x4D>": 78, + "<0x4E>": 79, + "<0x4F>": 80, + "<0x50>": 81, + "<0x51>": 82, + "<0x52>": 83, + "<0x53>": 84, + "<0x54>": 85, + "<0x55>": 86, + "<0x56>": 87, + "<0x57>": 88, + "<0x58>": 89, + "<0x59>": 90, + "<0x5A>": 91, + "<0x5B>": 92, + "<0x5C>": 93, + "<0x5D>": 94, + "<0x5E>": 95, + "<0x5F>": 96, + "<0x60>": 97, + "<0x61>": 98, + "<0x62>": 99, + "<0x63>": 100, + "<0x64>": 101, + "<0x65>": 102, + "<0x66>": 103, + "<0x67>": 104, + "<0x68>": 105, + "<0x69>": 106, + "<0x6A>": 107, + "<0x6B>": 108, + "<0x6C>": 109, + "<0x6D>": 110, + "<0x6E>": 111, + "<0x6F>": 112, + "<0x70>": 113, + "<0x71>": 114, + "<0x72>": 115, + "<0x73>": 116, + "<0x74>": 117, + "<0x75>": 118, + "<0x76>": 119, + "<0x77>": 120, + "<0x78>": 121, + "<0x79>": 122, + "<0x7A>": 123, + "<0x7B>": 124, + "<0x7C>": 125, + "<0x7D>": 126, + "<0x7E>": 127, + "<0x7F>": 128, + "<0x80>": 129, + "<0x81>": 130, + "<0x82>": 131, + "<0x83>": 132, + "<0x84>": 133, + "<0x85>": 134, + "<0x86>": 135, + "<0x87>": 136, + "<0x88>": 137, + "<0x89>": 138, + "<0x8A>": 139, + "<0x8B>": 140, + "<0x8C>": 141, + "<0x8D>": 142, + "<0x8E>": 143, + "<0x8F>": 144, + "<0x90>": 145, + "<0x91>": 146, + "<0x92>": 147, + "<0x93>": 148, + "<0x94>": 149, + "<0x95>": 150, + "<0x96>": 151, + "<0x97>": 152, + "<0x98>": 153, + "<0x99>": 154, + "<0x9A>": 155, + "<0x9B>": 156, + "<0x9C>": 157, + "<0x9D>": 158, + "<0x9E>": 159, + "<0x9F>": 160, + "<0xA0>": 161, + "<0xA1>": 162, + "<0xA2>": 163, + "<0xA3>": 164, + "<0xA4>": 165, + "<0xA5>": 166, + "<0xA6>": 167, + "<0xA7>": 168, + "<0xA8>": 169, + "<0xA9>": 170, + "<0xAA>": 171, + "<0xAB>": 172, + "<0xAC>": 173, + "<0xAE>": 174, + "<0xAF>": 175, + "<0xB0>": 176, + "<0xB1>": 177, + "<0xB2>": 178, + "<0xB3>": 179, + "<0xB4>": 180, + "<0xB5>": 181, + "<0xB6>": 182, + "<0xB7>": 183, + "<0xB9>": 184, + "<0xBA>": 185, + "<0xBB>": 186, + "<0xBC>": 187, + "<0xBD>": 188, + "<0xBE>": 189, + "<0xBF>": 190, + "<0xC0>": 191, + "<0xC1>": 192, + "<0xC2>": 193, + "<0xC3>": 194, + "<0xC4>": 195, + "<0xC5>": 196, + "<0xC6>": 197, + "<0xC7>": 198, + "<0xC8>": 199, + "<0xC9>": 200, + "<0xCA>": 201, + "<0xCB>": 202, + "<0xCC>": 203, + "<0xCD>": 204, + "<0xCE>": 205, + "<0xCF>": 206, + "<0xD0>": 207, + "<0xD1>": 208, + "<0xD2>": 209, + "<0xD3>": 210, + "<0xD4>": 211, + "<0xD5>": 212, + "<0xD6>": 213, + "<0xD7>": 214, + "<0xD8>": 215, + "<0xD9>": 216, + "<0xDA>": 217, + "<0xDB>": 218, + "<0xDC>": 219, + "<0xDD>": 220, + "<0xDE>": 221, + "<0xDF>": 222, + "<0xE0>": 223, + "<0xE1>": 224, + "<0xE2>": 225, + "<0xE3>": 226, + "<0xE5>": 227, + "<0xE6>": 228, + "<0xE7>": 229, + "<0xE8>": 230, + "<0xE9>": 231, + "<0xEA>": 232, + "<0xEB>": 233, + "<0xEC>": 234, + "<0xED>": 235, + "<0xEE>": 236, + "<0xEF>": 237, + "<0xF0>": 238, + "<0xF1>": 239, + "<0xF2>": 240, + "<0xF3>": 241, + "<0xF4>": 242, + "<0xF5>": 243, + "<0xF6>": 244, + "<0xF7>": 245, + "<0xF8>": 246, + "<0xF9>": 247, + "<0xFA>": 248, + "<0xFB>": 249, + "<0xFC>": 250, + "<0xFD>": 251, + "<0xFE>": 252, + "<0xFF>": 253 + }, + "merges": [], + "flags": { + "byte_fallback": true + }, + "expected": { + "a": [ + "a" + ], + "az": [ + "a", + "<0x7A>" + ], + "中": [], + "é": [ + "<0xC3>", + "<0xA9>" + ], + "a中a": [ + "a", + "a" + ], + "\t": [ + "<0x09>" + ], + "\n": [ + "<0x0A>" + ], + "\u0001": [ + "<0x01>" + ], + "\u000f": [ + "<0x0F>" + ] + } + }, + { + "name": "bare_ascii_absent_but_byte_token_present", + "why": "byte fallback also fires for plain ASCII when the bare char is missing", + "vocab": { + "": 0, + "<0x61>": 1, + "<0x62>": 2, + "<0x63>": 3 + }, + "merges": [], + "flags": { + "unk_token": "", + "byte_fallback": true + }, + "expected": { + "abc": [ + "<0x61>", + "<0x62>", + "<0x63>" + ], + "a": [ + "<0x61>" + ], + "abd": [ + "<0x61>", + "<0x62>", + "" + ] + } + }, + { + "name": "bf_partial_bytes", + "why": "all-or-nothing per character, and the pending-unk slot is NOT flushed by a byte-fallback hit", + "vocab": { + "a": 0, + "": 1, + "<0x00>": 2, + "<0x01>": 3, + "<0x02>": 4, + "<0x03>": 5, + "<0x04>": 6, + "<0x05>": 7, + "<0x06>": 8, + "<0x07>": 9, + "<0x08>": 10, + "<0x09>": 11, + "<0x0A>": 12, + "<0x0B>": 13, + "<0x0C>": 14, + "<0x0D>": 15, + "<0x0E>": 16, + "<0x0F>": 17, + "<0x10>": 18, + "<0x11>": 19, + "<0x12>": 20, + "<0x13>": 21, + "<0x14>": 22, + "<0x15>": 23, + "<0x16>": 24, + "<0x17>": 25, + "<0x18>": 26, + "<0x19>": 27, + "<0x1A>": 28, + "<0x1B>": 29, + "<0x1C>": 30, + "<0x1D>": 31, + "<0x1E>": 32, + "<0x1F>": 33, + "<0x20>": 34, + "<0x21>": 35, + "<0x22>": 36, + "<0x23>": 37, + "<0x24>": 38, + "<0x25>": 39, + "<0x26>": 40, + "<0x27>": 41, + "<0x28>": 42, + "<0x29>": 43, + "<0x2A>": 44, + "<0x2B>": 45, + "<0x2C>": 46, + "<0x2D>": 47, + "<0x2E>": 48, + "<0x2F>": 49, + "<0x30>": 50, + "<0x31>": 51, + "<0x32>": 52, + "<0x33>": 53, + "<0x34>": 54, + "<0x35>": 55, + "<0x36>": 56, + "<0x37>": 57, + "<0x38>": 58, + "<0x39>": 59, + "<0x3A>": 60, + "<0x3B>": 61, + "<0x3C>": 62, + "<0x3D>": 63, + "<0x3E>": 64, + "<0x3F>": 65, + "<0x40>": 66, + "<0x41>": 67, + "<0x42>": 68, + "<0x43>": 69, + "<0x44>": 70, + "<0x45>": 71, + "<0x46>": 72, + "<0x47>": 73, + "<0x48>": 74, + "<0x49>": 75, + "<0x4A>": 76, + "<0x4B>": 77, + "<0x4C>": 78, + "<0x4D>": 79, + "<0x4E>": 80, + "<0x4F>": 81, + "<0x50>": 82, + "<0x51>": 83, + "<0x52>": 84, + "<0x53>": 85, + "<0x54>": 86, + "<0x55>": 87, + "<0x56>": 88, + "<0x57>": 89, + "<0x58>": 90, + "<0x59>": 91, + "<0x5A>": 92, + "<0x5B>": 93, + "<0x5C>": 94, + "<0x5D>": 95, + "<0x5E>": 96, + "<0x5F>": 97, + "<0x60>": 98, + "<0x61>": 99, + "<0x62>": 100, + "<0x63>": 101, + "<0x64>": 102, + "<0x65>": 103, + "<0x66>": 104, + "<0x67>": 105, + "<0x68>": 106, + "<0x69>": 107, + "<0x6A>": 108, + "<0x6B>": 109, + "<0x6C>": 110, + "<0x6D>": 111, + "<0x6E>": 112, + "<0x6F>": 113, + "<0x70>": 114, + "<0x71>": 115, + "<0x72>": 116, + "<0x73>": 117, + "<0x74>": 118, + "<0x75>": 119, + "<0x76>": 120, + "<0x77>": 121, + "<0x78>": 122, + "<0x79>": 123, + "<0x7A>": 124, + "<0x7B>": 125, + "<0x7C>": 126, + "<0x7D>": 127, + "<0x7E>": 128, + "<0x7F>": 129, + "<0x80>": 130, + "<0x81>": 131, + "<0x82>": 132, + "<0x83>": 133, + "<0x84>": 134, + "<0x85>": 135, + "<0x86>": 136, + "<0x87>": 137, + "<0x88>": 138, + "<0x89>": 139, + "<0x8A>": 140, + "<0x8B>": 141, + "<0x8C>": 142, + "<0x8D>": 143, + "<0x8E>": 144, + "<0x8F>": 145, + "<0x90>": 146, + "<0x91>": 147, + "<0x92>": 148, + "<0x93>": 149, + "<0x94>": 150, + "<0x95>": 151, + "<0x96>": 152, + "<0x97>": 153, + "<0x98>": 154, + "<0x99>": 155, + "<0x9A>": 156, + "<0x9B>": 157, + "<0x9C>": 158, + "<0x9D>": 159, + "<0x9E>": 160, + "<0x9F>": 161, + "<0xA0>": 162, + "<0xA1>": 163, + "<0xA2>": 164, + "<0xA3>": 165, + "<0xA4>": 166, + "<0xA5>": 167, + "<0xA6>": 168, + "<0xA7>": 169, + "<0xA8>": 170, + "<0xA9>": 171, + "<0xAA>": 172, + "<0xAB>": 173, + "<0xAC>": 174, + "<0xAD>": 175, + "<0xAE>": 176, + "<0xAF>": 177, + "<0xB0>": 178, + "<0xB1>": 179, + "<0xB2>": 180, + "<0xB3>": 181, + "<0xB4>": 182, + "<0xB5>": 183, + "<0xB6>": 184, + "<0xB7>": 185, + "<0xB8>": 186, + "<0xB9>": 187, + "<0xBA>": 188, + "<0xBB>": 189, + "<0xBC>": 190, + "<0xBD>": 191, + "<0xBE>": 192, + "<0xBF>": 193, + "<0xC0>": 194, + "<0xC1>": 195, + "<0xC2>": 196, + "<0xC3>": 197, + "<0xC4>": 198, + "<0xC5>": 199, + "<0xC6>": 200, + "<0xC7>": 201, + "<0xC8>": 202, + "<0xC9>": 203, + "<0xCA>": 204, + "<0xCB>": 205, + "<0xCC>": 206, + "<0xCD>": 207, + "<0xCE>": 208, + "<0xCF>": 209, + "<0xD0>": 210, + "<0xD1>": 211, + "<0xD2>": 212, + "<0xD3>": 213, + "<0xD4>": 214, + "<0xD5>": 215, + "<0xD6>": 216, + "<0xD7>": 217, + "<0xD8>": 218, + "<0xD9>": 219, + "<0xDA>": 220, + "<0xDB>": 221, + "<0xDC>": 222, + "<0xDD>": 223, + "<0xDE>": 224, + "<0xDF>": 225, + "<0xE0>": 226, + "<0xE1>": 227, + "<0xE2>": 228, + "<0xE3>": 229, + "<0xE5>": 230, + "<0xE6>": 231, + "<0xE7>": 232, + "<0xE8>": 233, + "<0xE9>": 234, + "<0xEA>": 235, + "<0xEB>": 236, + "<0xEC>": 237, + "<0xED>": 238, + "<0xEE>": 239, + "<0xEF>": 240, + "<0xF0>": 241, + "<0xF1>": 242, + "<0xF2>": 243, + "<0xF3>": 244, + "<0xF4>": 245, + "<0xF5>": 246, + "<0xF6>": 247, + "<0xF7>": 248, + "<0xF8>": 249, + "<0xF9>": 250, + "<0xFA>": 251, + "<0xFB>": 252, + "<0xFC>": 253, + "<0xFD>": 254, + "<0xFE>": 255, + "<0xFF>": 256 + }, + "merges": [], + "flags": { + "unk_token": "", + "byte_fallback": true + }, + "expected": { + "中": [ + "" + ], + "中é": [ + "<0xC3>", + "<0xA9>", + "" + ], + "é中": [ + "<0xC3>", + "<0xA9>", + "" + ], + "中中é": [ + "", + "<0xC3>", + "<0xA9>", + "" + ], + "中é中": [ + "<0xC3>", + "<0xA9>", + "", + "" + ], + "中中": [ + "", + "" + ], + "aé中": [ + "a", + "<0xC3>", + "<0xA9>", + "" + ], + "中a": [ + "", + "a" + ] + } + }, + { + "name": "bf_partial_bytes_fused", + "why": "same slot behaviour with coalescing on top", + "vocab": { + "a": 0, + "": 1, + "<0x00>": 2, + "<0x01>": 3, + "<0x02>": 4, + "<0x03>": 5, + "<0x04>": 6, + "<0x05>": 7, + "<0x06>": 8, + "<0x07>": 9, + "<0x08>": 10, + "<0x09>": 11, + "<0x0A>": 12, + "<0x0B>": 13, + "<0x0C>": 14, + "<0x0D>": 15, + "<0x0E>": 16, + "<0x0F>": 17, + "<0x10>": 18, + "<0x11>": 19, + "<0x12>": 20, + "<0x13>": 21, + "<0x14>": 22, + "<0x15>": 23, + "<0x16>": 24, + "<0x17>": 25, + "<0x18>": 26, + "<0x19>": 27, + "<0x1A>": 28, + "<0x1B>": 29, + "<0x1C>": 30, + "<0x1D>": 31, + "<0x1E>": 32, + "<0x1F>": 33, + "<0x20>": 34, + "<0x21>": 35, + "<0x22>": 36, + "<0x23>": 37, + "<0x24>": 38, + "<0x25>": 39, + "<0x26>": 40, + "<0x27>": 41, + "<0x28>": 42, + "<0x29>": 43, + "<0x2A>": 44, + "<0x2B>": 45, + "<0x2C>": 46, + "<0x2D>": 47, + "<0x2E>": 48, + "<0x2F>": 49, + "<0x30>": 50, + "<0x31>": 51, + "<0x32>": 52, + "<0x33>": 53, + "<0x34>": 54, + "<0x35>": 55, + "<0x36>": 56, + "<0x37>": 57, + "<0x38>": 58, + "<0x39>": 59, + "<0x3A>": 60, + "<0x3B>": 61, + "<0x3C>": 62, + "<0x3D>": 63, + "<0x3E>": 64, + "<0x3F>": 65, + "<0x40>": 66, + "<0x41>": 67, + "<0x42>": 68, + "<0x43>": 69, + "<0x44>": 70, + "<0x45>": 71, + "<0x46>": 72, + "<0x47>": 73, + "<0x48>": 74, + "<0x49>": 75, + "<0x4A>": 76, + "<0x4B>": 77, + "<0x4C>": 78, + "<0x4D>": 79, + "<0x4E>": 80, + "<0x4F>": 81, + "<0x50>": 82, + "<0x51>": 83, + "<0x52>": 84, + "<0x53>": 85, + "<0x54>": 86, + "<0x55>": 87, + "<0x56>": 88, + "<0x57>": 89, + "<0x58>": 90, + "<0x59>": 91, + "<0x5A>": 92, + "<0x5B>": 93, + "<0x5C>": 94, + "<0x5D>": 95, + "<0x5E>": 96, + "<0x5F>": 97, + "<0x60>": 98, + "<0x61>": 99, + "<0x62>": 100, + "<0x63>": 101, + "<0x64>": 102, + "<0x65>": 103, + "<0x66>": 104, + "<0x67>": 105, + "<0x68>": 106, + "<0x69>": 107, + "<0x6A>": 108, + "<0x6B>": 109, + "<0x6C>": 110, + "<0x6D>": 111, + "<0x6E>": 112, + "<0x6F>": 113, + "<0x70>": 114, + "<0x71>": 115, + "<0x72>": 116, + "<0x73>": 117, + "<0x74>": 118, + "<0x75>": 119, + "<0x76>": 120, + "<0x77>": 121, + "<0x78>": 122, + "<0x79>": 123, + "<0x7A>": 124, + "<0x7B>": 125, + "<0x7C>": 126, + "<0x7D>": 127, + "<0x7E>": 128, + "<0x7F>": 129, + "<0x80>": 130, + "<0x81>": 131, + "<0x82>": 132, + "<0x83>": 133, + "<0x84>": 134, + "<0x85>": 135, + "<0x86>": 136, + "<0x87>": 137, + "<0x88>": 138, + "<0x89>": 139, + "<0x8A>": 140, + "<0x8B>": 141, + "<0x8C>": 142, + "<0x8D>": 143, + "<0x8E>": 144, + "<0x8F>": 145, + "<0x90>": 146, + "<0x91>": 147, + "<0x92>": 148, + "<0x93>": 149, + "<0x94>": 150, + "<0x95>": 151, + "<0x96>": 152, + "<0x97>": 153, + "<0x98>": 154, + "<0x99>": 155, + "<0x9A>": 156, + "<0x9B>": 157, + "<0x9C>": 158, + "<0x9D>": 159, + "<0x9E>": 160, + "<0x9F>": 161, + "<0xA0>": 162, + "<0xA1>": 163, + "<0xA2>": 164, + "<0xA3>": 165, + "<0xA4>": 166, + "<0xA5>": 167, + "<0xA6>": 168, + "<0xA7>": 169, + "<0xA8>": 170, + "<0xA9>": 171, + "<0xAA>": 172, + "<0xAB>": 173, + "<0xAC>": 174, + "<0xAD>": 175, + "<0xAE>": 176, + "<0xAF>": 177, + "<0xB0>": 178, + "<0xB1>": 179, + "<0xB2>": 180, + "<0xB3>": 181, + "<0xB4>": 182, + "<0xB5>": 183, + "<0xB6>": 184, + "<0xB7>": 185, + "<0xB8>": 186, + "<0xB9>": 187, + "<0xBA>": 188, + "<0xBB>": 189, + "<0xBC>": 190, + "<0xBD>": 191, + "<0xBE>": 192, + "<0xBF>": 193, + "<0xC0>": 194, + "<0xC1>": 195, + "<0xC2>": 196, + "<0xC3>": 197, + "<0xC4>": 198, + "<0xC5>": 199, + "<0xC6>": 200, + "<0xC7>": 201, + "<0xC8>": 202, + "<0xC9>": 203, + "<0xCA>": 204, + "<0xCB>": 205, + "<0xCC>": 206, + "<0xCD>": 207, + "<0xCE>": 208, + "<0xCF>": 209, + "<0xD0>": 210, + "<0xD1>": 211, + "<0xD2>": 212, + "<0xD3>": 213, + "<0xD4>": 214, + "<0xD5>": 215, + "<0xD6>": 216, + "<0xD7>": 217, + "<0xD8>": 218, + "<0xD9>": 219, + "<0xDA>": 220, + "<0xDB>": 221, + "<0xDC>": 222, + "<0xDD>": 223, + "<0xDE>": 224, + "<0xDF>": 225, + "<0xE0>": 226, + "<0xE1>": 227, + "<0xE2>": 228, + "<0xE3>": 229, + "<0xE5>": 230, + "<0xE6>": 231, + "<0xE7>": 232, + "<0xE8>": 233, + "<0xE9>": 234, + "<0xEA>": 235, + "<0xEB>": 236, + "<0xEC>": 237, + "<0xED>": 238, + "<0xEE>": 239, + "<0xEF>": 240, + "<0xF0>": 241, + "<0xF1>": 242, + "<0xF2>": 243, + "<0xF3>": 244, + "<0xF4>": 245, + "<0xF5>": 246, + "<0xF6>": 247, + "<0xF7>": 248, + "<0xF8>": 249, + "<0xF9>": 250, + "<0xFA>": 251, + "<0xFB>": 252, + "<0xFC>": 253, + "<0xFD>": 254, + "<0xFE>": 255, + "<0xFF>": 256 + }, + "merges": [], + "flags": { + "unk_token": "", + "byte_fallback": true, + "fuse_unk": true + }, + "expected": { + "中": [ + "" + ], + "中é": [ + "<0xC3>", + "<0xA9>", + "" + ], + "é中": [ + "<0xC3>", + "<0xA9>", + "" + ], + "中中é": [ + "<0xC3>", + "<0xA9>", + "" + ], + "中é中": [ + "<0xC3>", + "<0xA9>", + "" + ], + "中中": [ + "" + ], + "aé中": [ + "a", + "<0xC3>", + "<0xA9>", + "" + ], + "中a": [ + "", + "a" + ] + } + }, + { + "name": "merge_over_byte_tokens", + "why": "byte-fallback tokens participate in merges, so resolution precedes merging", + "vocab": { + "a": 0, + "": 1, + "<0xE4>": 2, + "<0xB8>": 3, + "<0xAD>": 4, + "<0xE4><0xB8>": 5, + "a<0xE4>": 6 + }, + "merges": [ + [ + "<0xE4>", + "<0xB8>" + ] + ], + "flags": { + "unk_token": "", + "byte_fallback": true + }, + "expected": { + "中": [ + "<0xE4><0xB8>", + "<0xAD>" + ], + "a中": [ + "a", + "<0xE4><0xB8>", + "<0xAD>" + ] + } + }, + { + "name": "merge_over_byte_tokens_rank_order", + "why": "merge rank over byte tokens changes the result", + "vocab": { + "a": 0, + "": 1, + "<0xE4>": 2, + "<0xB8>": 3, + "<0xAD>": 4, + "<0xE4><0xB8>": 5, + "a<0xE4>": 6 + }, + "merges": [ + [ + "a", + "<0xE4>" + ], + [ + "<0xE4>", + "<0xB8>" + ] + ], + "flags": { + "unk_token": "", + "byte_fallback": true + }, + "expected": { + "中": [ + "<0xE4><0xB8>", + "<0xAD>" + ], + "a中": [ + "a<0xE4>", + "<0xB8>", + "<0xAD>" + ] + } + }, + { + "name": "merge_over_unk_tokens", + "why": "generated unk tokens participate in merges too", + "vocab": { + "a": 0, + "": 1, + "": 2 + }, + "merges": [ + [ + "", + "" + ] + ], + "flags": { + "unk_token": "" + }, + "expected": { + "zz": [ + "" + ], + "azza": [ + "a", + "", + "a" + ], + "z": [ + "" + ], + "zzz": [ + "", + "" + ] + } + }, + { + "name": "fuse_unk_without_unk_token", + "why": "fuse_unk is inert with no unk token to fuse", + "vocab": { + "a": 0, + "b": 1 + }, + "merges": [], + "flags": { + "fuse_unk": true + }, + "expected": { + "azza": [ + "a", + "a" + ], + "zz": [], + "ab": [ + "a", + "b" + ] + } + }, + { + "name": "ignore_merges_falls_through_to_unk", + "why": "a piece absent from vocab still takes the normal resolve path", + "vocab": { + "a": 0, + "b": 1, + "ab": 2, + "": 3 + }, + "merges": [ + [ + "a", + "b" + ] + ], + "flags": { + "ignore_merges": true, + "unk_token": "" + }, + "expected": { + "ab": [ + "ab" + ], + "abz": [ + "ab", + "" + ], + "zz": [ + "", + "" + ], + "a": [ + "a" + ] + } + } + ] +} diff --git a/tests/test_bpe_flags.py b/tests/test_bpe_flags.py new file mode 100644 index 0000000..3e29c4e --- /dev/null +++ b/tests/test_bpe_flags.py @@ -0,0 +1,91 @@ +"""BPE option behaviour, checked against recorded HuggingFace output. + +The expectations in ``fixtures/hf_bpe_flags.json`` were measured by running real +``huggingface/tokenizers``; see ``scripts/gen_hf_flag_fixtures.py``. This module +reads that recording using only the standard library, so the suite keeps working +without any third-party package installed. +""" + +import json +from pathlib import Path + +import pytest + +from ubi_tokenizer.errors import ModelError +from ubi_tokenizer.models.bpe import BPE + +FIXTURE = Path(__file__).parent / "fixtures" / "hf_bpe_flags.json" + + +def _load_cases(): + # A missing fixture is a hard failure, never a skip: silently skipping would + # remove all HuggingFace parity coverage without anyone noticing. + if not FIXTURE.exists(): + raise AssertionError( + f"missing ground-truth fixture {FIXTURE}; " + "regenerate with scripts/gen_hf_flag_fixtures.py" + ) + return json.loads(FIXTURE.read_text(encoding="utf-8"))["cases"] + + +CASES = _load_cases() + + +def _model(case): + return BPE( + vocab=dict(case["vocab"]), + merges=[tuple(pair) for pair in case["merges"]], + **case["flags"], + ) + + +def _expectations(): + for case in CASES: + for text, expected in case["expected"].items(): + yield pytest.param(case, text, expected, id=f"{case['name']}-{text!r}") + + +@pytest.mark.parametrize(("case", "text", "expected"), list(_expectations())) +def test_matches_huggingface(case, text, expected): + assert _model(case).tokenize(text) == expected, case["why"] + + +def test_fixture_records_its_ground_truth_version(): + payload = json.loads(FIXTURE.read_text(encoding="utf-8")) + assert payload["_ground_truth"].startswith("huggingface tokenizers ") + + +def test_ignore_merges_is_actually_discriminating(): + # Guards the fixture set itself: if these two cases ever agree, the + # ignore_merges coverage has quietly become vacuous. + off = next(c for c in CASES if c["name"] == "ignore_merges_off_control") + on = next(c for c in CASES if c["name"] == "ignore_merges_discriminating") + assert off["expected"] != on["expected"] + + +# -- behaviour with no HuggingFace counterpart ------------------------------ + + +def test_unk_token_absent_from_vocab_raises(): + # HuggingFace raises lazily at tokenize time rather than at construction, + # and so do we. Constructing must stay quiet. + model = BPE(vocab={"a": 0}, merges=[], unk_token="") + assert model.tokenize("a") == ["a"] + with pytest.raises(ModelError, match=""): + model.tokenize("az") + + +def test_byte_fallback_hit_still_works_when_unk_is_absent_from_vocab(): + # Only the unk path needs the unk token, so a pure byte-fallback hit must + # not be dragged down by a missing unk. + vocab = {f"<0x{value:02X}>": value for value in range(256)} + model = BPE(vocab=vocab, merges=[], unk_token="", byte_fallback=True) + assert model.tokenize("a") == ["<0x61>"] + + +def test_lone_surrogate_propagates_encoding_error(): + # Pure Python can be handed a lone surrogate where HuggingFace's binding + # layer would already have rejected it. Propagating is deliberate. + model = BPE(vocab={"a": 0}, merges=[], byte_fallback=True) + with pytest.raises(UnicodeEncodeError): + model.tokenize("\ud800")