diff --git a/docs/components.md b/docs/components.md index 77c10a9..b6f009d 100644 --- a/docs/components.md +++ b/docs/components.md @@ -37,8 +37,38 @@ byte-mapped space. Turns one byte-mapped piece into subword tokens. - `BPE(vocab, merges, *, unk_token=None, ...)` — byte-pair encoding. `tokenize` - greedily applies the lowest-rank applicable merge until none remain (identical - to the UbiTok reference runtime). + resolves each character onto vocabulary tokens, then greedily applies the + lowest-rank applicable merge until none remain (identical to the UbiTok + reference runtime). + +Resolution happens **before** merging, because the tokens it produces take part +in merges themselves — a vocabulary containing `<0xE4><0xB8>` plus a merge over +its two halves tokenizes `中` as `['<0xE4><0xB8>', '<0xAD>']`. + +| Option | Effect | +| --- | --- | +| `unk_token` | out-of-vocabulary characters become this token. Without it they are **dropped**, matching HuggingFace | +| `fuse_unk` | consecutive unresolvable characters collapse into a single unk | +| `byte_fallback` | an out-of-vocabulary character becomes its UTF-8 bytes as `<0xXX>` tokens (uppercase, zero-padded). All-or-nothing per character: one missing byte token sends the whole character to unk | +| `ignore_merges` | a piece that is already a vocabulary entry is emitted whole, skipping merges. Required by GPT-4o / Llama-3 tokenizers | + +Setting `unk_token` to a string that is absent from the vocabulary is allowed at +construction and raises `ModelError` only if unresolvable input actually arrives, +which is when HuggingFace raises too. + +### Two behaviours worth knowing + +**A pending unk survives a byte-fallback hit.** HuggingFace defers at most one +unresolved character, and a successful byte expansion does not flush it. With +`<0xE4>` missing from the vocabulary, `'中é'` gives +`['<0xC3>', '<0xA9>', '']` — the unk for `中` lands *after* the bytes of `é`. +This is likely an upstream quirk, but byte-parity with HuggingFace is the point +of this package, so it is reproduced rather than corrected. + +**`byte_fallback` is encode-only.** The `ByteLevel` decoder maps each character +through the byte table, so it renders `<0xE4>` as the six ASCII characters of its +own name rather than as part of `中`. Round-tripping byte-fallback output needs a +`ByteFallback` decoder, which does not exist yet. ## PostProcessor (`ubi_tokenizer.processors`) @@ -61,3 +91,24 @@ Reconstructs text from tokens. - `BpeTrainer(vocab_size, min_frequency, special_tokens=None, ...)` — in-memory greedy trainer. For large/distributed training use UbiTok; this is for small corpora and tests. + +## Unimplemented options + +`BPE` and `BpeTrainer` accept the full HuggingFace option set so that a +`tokenizer.json` round-trips unchanged. Options that are not implemented raise +`UnsupportedFeatureError` when set to a value that would change tokenization, +rather than being accepted and ignored: + +| Option | Accepted values | Rejected values | +| --- | --- | --- | +| `dropout` | `None`, `0.0` | anything in `(0, 1]` — outside `[0, 1]` raises `ModelError` | +| `continuing_subword_prefix` | `None`, `""` | any non-empty string | +| `end_of_word_suffix` | `None`, `""` | any non-empty string | + +The accepted values are genuine no-ops in HuggingFace as well, so files that +tokenize correctly today keep loading — GPT-2-lineage exports ship `""` for both +affix options. + +`cache_capacity` is accepted and retained but not yet used. It is not rejected +because a cache is a pure optimization: its absence cannot change output. +HuggingFace does not serialize it into `tokenizer.json`, so `to_dict()` omits it. diff --git a/pyproject.toml b/pyproject.toml index 4b8daf4..5e80d97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,10 @@ dependencies = [] [project.optional-dependencies] transcription = [] +# Only needed to regenerate tests/fixtures/hf_bpe_flags.json; the test suite +# itself runs on the standard library alone. The tokenizers pin is deliberate: +# the recorded ground truth includes version-sensitive behaviour. +dev = ["pytest", "tokenizers==0.23.1"] [project.scripts] ubi-tokenizer = "ubi_tokenizer.__main__:main" diff --git a/scripts/gen_hf_flag_fixtures.py b/scripts/gen_hf_flag_fixtures.py new file mode 100644 index 0000000..157a088 --- /dev/null +++ b/scripts/gen_hf_flag_fixtures.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Regenerate the HuggingFace ground-truth fixtures for BPE option behaviour. + +``tests/test_bpe_flags.py`` checks ``ubi_tokenizer``'s BPE against recorded +HuggingFace output. This script produces that recording, so the expectations are +measured rather than derived from reading the Rust source. + +It is the only part of the test setup that needs a third-party package, and it is +run by hand -- the test suite itself stays standard-library only, which is the +point of the whole project. Install the dev extra first: + + pip install -e ".[dev]" + python scripts/gen_hf_flag_fixtures.py + +The ``tokenizers`` version is pinned in the dev extra and recorded in the output. +Several behaviours captured here are quirky enough (see ``bf_partial_bytes``) +that regenerating against a different version could silently move ground truth. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +try: + import tokenizers + from tokenizers import models +except ImportError: # pragma: no cover - developer-facing script + raise SystemExit( + 'huggingface tokenizers is required to regenerate fixtures: pip install -e ".[dev]"' + ) + +OUTPUT = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / "hf_bpe_flags.json" + +BYTE_TOKENS = [f"<0x{value:02X}>" for value in range(256)] + + +def vocab_of(*groups: list[str]) -> dict[str, int]: + """Assign ids in listed order, skipping duplicates.""" + vocab: dict[str, int] = {} + for group in groups: + for token in group: + if token not in vocab: + vocab[token] = len(vocab) + return vocab + + +def bytes_except(*excluded: str) -> list[str]: + return [token for token in BYTE_TOKENS if token not in excluded] + + +# Inputs reused across the byte-fallback cases. "\t" (0x09), "\n" (0x0A) and +# "\x01" exercise bytes below 0x10, where an implementation that produced +# uppercase-but-not-zero-padded hex would otherwise pass unnoticed -- every +# CJK/accented character has bytes >= 0x80 and so is always two hex digits. +LOW_BYTE_INPUTS = ["\t", "\n", "\x01", "\x0f"] + +CASES: list[dict[str, Any]] = [ + { + "name": "baseline_no_flags", + "why": "merge loop must not regress", + "vocab": vocab_of(["a", "b", "c", "ab", "abc"]), + "merges": [("a", "b"), ("ab", "c")], + "flags": {}, + "inputs": ["abc", "ab", "a", "abz", "zz", "az", ""], + }, + { + "name": "ignore_merges_off_control", + "why": "control for the case below; proves the flag is what changes output", + "vocab": vocab_of(["x", "y", "xy", "z", "xz"]), + "merges": [("x", "z")], + "flags": {}, + "inputs": ["xy", "xz", "xyz", "xyxy", ""], + }, + { + "name": "ignore_merges_discriminating", + "why": "'xy' is in vocab but no merge reaches it, so the flag alone decides", + "vocab": vocab_of(["x", "y", "xy", "z", "xz"]), + "merges": [("x", "z")], + "flags": {"ignore_merges": True}, + "inputs": ["xy", "xz", "xyz", "xyxy", ""], + }, + { + "name": "drop_when_no_unk", + "why": "out-of-vocab characters are dropped, not turned into an invalid id", + "vocab": vocab_of(["a", "b"]), + "merges": [], + "flags": {}, + "inputs": ["zz", "azb", "az", "za", "中"], + }, + { + "name": "unk_only", + "why": "one unk per out-of-vocab character", + "vocab": vocab_of(["a", "b", "ab", ""]), + "merges": [("a", "b")], + "flags": {"unk_token": ""}, + "inputs": ["ab", "az", "zz", "az z", "中", ""], + }, + { + "name": "unk_fused", + "why": "consecutive pending unks coalesce", + "vocab": vocab_of(["a", "b", "ab", ""]), + "merges": [("a", "b")], + "flags": {"unk_token": "", "fuse_unk": True}, + "inputs": ["ab", "az", "zz", "az z", "中", "azza"], + }, + { + "name": "unk_token_present_in_vocab", + "why": "an in-vocab unk string must not swallow a following genuine unk", + "vocab": vocab_of(["a", "U"]), + "merges": [], + "flags": {"unk_token": "U", "fuse_unk": True}, + "inputs": ["Uz", "zU", "aUza", "UU"], + }, + { + "name": "byte_fallback_with_unk", + "why": "byte expansion, including bytes below 0x10", + "vocab": vocab_of(["a", ""], BYTE_TOKENS), + "merges": [], + "flags": {"unk_token": "", "byte_fallback": True}, + "inputs": ["a", "az", "中", "é", "abc", "€", *LOW_BYTE_INPUTS], + }, + { + "name": "byte_fallback_without_unk", + "why": "no unk to fall back on; misses are dropped", + "vocab": vocab_of(["a"], bytes_except("<0xE4>", "<0xB8>", "<0xAD>")), + "merges": [], + "flags": {"byte_fallback": True}, + "inputs": ["a", "az", "中", "é", "a中a", *LOW_BYTE_INPUTS], + }, + { + "name": "bare_ascii_absent_but_byte_token_present", + "why": "byte fallback also fires for plain ASCII when the bare char is missing", + "vocab": vocab_of([""], ["<0x61>", "<0x62>", "<0x63>"]), + "merges": [], + "flags": {"unk_token": "", "byte_fallback": True}, + "inputs": ["abc", "a", "abd"], + }, + { + "name": "bf_partial_bytes", + "why": "all-or-nothing per character, and the pending-unk slot is NOT flushed by a byte-fallback hit", + "vocab": vocab_of(["a", ""], bytes_except("<0xE4>")), + "merges": [], + "flags": {"unk_token": "", "byte_fallback": True}, + "inputs": ["中", "中é", "é中", "中中é", "中é中", "中中", "aé中", "中a"], + }, + { + "name": "bf_partial_bytes_fused", + "why": "same slot behaviour with coalescing on top", + "vocab": vocab_of(["a", ""], bytes_except("<0xE4>")), + "merges": [], + "flags": {"unk_token": "", "byte_fallback": True, "fuse_unk": True}, + "inputs": ["中", "中é", "é中", "中中é", "中é中", "中中", "aé中", "中a"], + }, + { + "name": "merge_over_byte_tokens", + "why": "byte-fallback tokens participate in merges, so resolution precedes merging", + "vocab": vocab_of( + ["a", "", "<0xE4>", "<0xB8>", "<0xAD>", "<0xE4><0xB8>", "a<0xE4>"] + ), + "merges": [("<0xE4>", "<0xB8>")], + "flags": {"unk_token": "", "byte_fallback": True}, + "inputs": ["中", "a中"], + }, + { + "name": "merge_over_byte_tokens_rank_order", + "why": "merge rank over byte tokens changes the result", + "vocab": vocab_of( + ["a", "", "<0xE4>", "<0xB8>", "<0xAD>", "<0xE4><0xB8>", "a<0xE4>"] + ), + "merges": [("a", "<0xE4>"), ("<0xE4>", "<0xB8>")], + "flags": {"unk_token": "", "byte_fallback": True}, + "inputs": ["中", "a中"], + }, + { + "name": "merge_over_unk_tokens", + "why": "generated unk tokens participate in merges too", + "vocab": vocab_of(["a", "", ""]), + "merges": [("", "")], + "flags": {"unk_token": ""}, + "inputs": ["zz", "azza", "z", "zzz"], + }, + { + "name": "fuse_unk_without_unk_token", + "why": "fuse_unk is inert with no unk token to fuse", + "vocab": vocab_of(["a", "b"]), + "merges": [], + "flags": {"fuse_unk": True}, + "inputs": ["azza", "zz", "ab"], + }, + { + "name": "ignore_merges_falls_through_to_unk", + "why": "a piece absent from vocab still takes the normal resolve path", + "vocab": vocab_of(["a", "b", "ab", ""]), + "merges": [("a", "b")], + "flags": {"ignore_merges": True, "unk_token": ""}, + "inputs": ["ab", "abz", "zz", "a"], + }, +] + + +def build(case: dict[str, Any]) -> dict[str, Any]: + model = models.BPE( + vocab=dict(case["vocab"]), + merges=[tuple(pair) for pair in case["merges"]], + **case["flags"], + ) + expected = {} + for text in case["inputs"]: + expected[text] = [token.value for token in model.tokenize(text)] + return { + "name": case["name"], + "why": case["why"], + "vocab": case["vocab"], + "merges": [list(pair) for pair in case["merges"]], + "flags": case["flags"], + "expected": expected, + } + + +def main() -> None: + payload = { + "_generated_by": "scripts/gen_hf_flag_fixtures.py", + "_ground_truth": f"huggingface tokenizers {tokenizers.__version__}", + "_warning": ( + "Do not hand-edit. Regenerate with the pinned tokenizers version; some " + "recorded behaviour is version-sensitive." + ), + "cases": [build(case) for case in CASES], + } + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + OUTPUT.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + total = sum(len(case["expected"]) for case in payload["cases"]) + print(f"wrote {OUTPUT.relative_to(Path.cwd())}: {len(payload['cases'])} cases, {total} inputs") + + +if __name__ == "__main__": + main() diff --git a/src/ubi_tokenizer/_validation.py b/src/ubi_tokenizer/_validation.py new file mode 100644 index 0000000..7ab5629 --- /dev/null +++ b/src/ubi_tokenizer/_validation.py @@ -0,0 +1,54 @@ +"""Shared rejection rules for recognized-but-unimplemented options. + +The BPE model and the BPE trainer both accept options that they store, and in +the model's case serialize, without ever acting on them. Silently accepting +such an option hands the caller wrong tokenization with no signal, so a value +that would change behaviour is rejected instead. + +The rule has to stay identical in both places, so it lives here rather than +being duplicated. +""" + +from __future__ import annotations + +from typing import Optional + +from ubi_tokenizer.errors import ModelError, UnsupportedFeatureError + +_HINT = ( + "recognized but not implemented by ubi_tokenizer. Drop the option, or use " + "huggingface/tokenizers if you need it." +) + + +def reject_unimplemented_affixes( + continuing_subword_prefix: Optional[str], + end_of_word_suffix: Optional[str], +) -> None: + """Reject affix options that would change tokenization. + + ``None`` and ``""`` are genuine no-ops in HuggingFace and stay accepted -- + GPT-2-lineage ``tokenizer.json`` files ship ``""`` for both, and rejecting + those would break loading files that tokenize correctly today. + """ + for name, value in ( + ("continuing_subword_prefix", continuing_subword_prefix), + ("end_of_word_suffix", end_of_word_suffix), + ): + if value: + raise UnsupportedFeatureError(f"{name}={value!r} is {_HINT}") + + +def reject_unimplemented_dropout(dropout: Optional[float]) -> None: + """Reject BPE dropout. + + ``None`` and ``0.0`` both mean "no dropout" and stay accepted. Values + outside ``[0, 1]`` are invalid rather than unimplemented, which is why they + raise :class:`ModelError`; HuggingFace rejects them at construction too. + """ + if dropout is None: + return + if not 0.0 <= dropout <= 1.0: + raise ModelError(f"dropout={dropout!r} must be between 0 and 1, inclusive") + if dropout > 0.0: + raise UnsupportedFeatureError(f"dropout={dropout!r} is {_HINT}") diff --git a/src/ubi_tokenizer/errors.py b/src/ubi_tokenizer/errors.py index b72daaf..6907d75 100644 --- a/src/ubi_tokenizer/errors.py +++ b/src/ubi_tokenizer/errors.py @@ -13,3 +13,12 @@ class DeserializationError(UbiTokenizerError): class ModelError(UbiTokenizerError): """Raised for invalid model state (missing tokens, bad merges, etc.).""" + + +class UnsupportedFeatureError(UbiTokenizerError): + """Raised when a requested option is recognized but not implemented. + + Distinct from :class:`ModelError`: the request is well-formed, it simply + cannot be honoured. Raised eagerly so the caller finds out when the + tokenizer is built rather than getting silently wrong tokenization. + """ diff --git a/src/ubi_tokenizer/models/bpe.py b/src/ubi_tokenizer/models/bpe.py index 4be8077..cf5280a 100644 --- a/src/ubi_tokenizer/models/bpe.py +++ b/src/ubi_tokenizer/models/bpe.py @@ -8,8 +8,15 @@ from __future__ import annotations +from collections.abc import Sequence from typing import Any, Optional +from ubi_tokenizer._validation import ( + reject_unimplemented_affixes, + reject_unimplemented_dropout, +) +from ubi_tokenizer.errors import ModelError + class Model: """Abstract base for tokenization models.""" @@ -47,8 +54,14 @@ def __init__( byte_fallback: bool = False, ignore_merges: bool = False, ) -> None: + reject_unimplemented_dropout(dropout) + reject_unimplemented_affixes(continuing_subword_prefix, end_of_word_suffix) self.vocab: dict[str, int] = dict(vocab) if vocab else {} self.merges: list[tuple[str, str]] = [tuple(m) for m in (merges or [])] + # Accepted rather than rejected: a cache is a pure optimization, so its + # absence cannot change output. HuggingFace does not serialize it into + # tokenizer.json either, so to_dict() stays faithful by omitting it. + self.cache_capacity = cache_capacity self.dropout = dropout self.unk_token = unk_token self.continuing_subword_prefix = continuing_subword_prefix @@ -75,10 +88,76 @@ def set_vocab_and_merges( self.merges = [tuple(m) for m in merges] self._rebuild() - # -- core merge logic (lifted from ByteLevelBPE._apply_bpe) ------------ + # -- tokenization ------------------------------------------------------ def tokenize(self, piece: str) -> list[str]: - """Apply BPE merges to one byte-mapped pre-token piece.""" - symbols = tuple(piece) + """Turn one pre-token piece into subword tokens.""" + if not piece: + return [] + # A whole piece that is already a vocabulary entry is emitted as-is with + # no merging. GPT-4o / Llama-3 tokenizers rely on this. + if self.ignore_merges and piece in self.vocab: + return [piece] + return self._apply_merges(self._resolve_symbols(piece)) + + def _resolve_symbols(self, piece: str) -> list[str]: + """Map each character onto vocabulary tokens, resolving unknowns. + + Runs *before* merging, because the tokens produced here -- byte-fallback + tokens and generated unks alike -- take part in merges themselves. A + vocabulary containing ``<0xE4><0xB8>`` with a merge over its two halves + tokenizes ``中`` as ``['<0xE4><0xB8>', '<0xAD>']``. + + Unresolvable characters are dropped when there is no unk token, which is + what HuggingFace does. + """ + symbols: list[str] = [] + # HuggingFace holds at most one deferred unk, and what flushes it is + # load-bearing; see the byte-fallback branch below. + pending_unk = False + for character in piece: + if character in self.vocab: + if pending_unk: + symbols.append(self._unk_token_or_raise()) + pending_unk = False + symbols.append(character) + continue + if self.byte_fallback: + byte_tokens = [ + f"<0x{value:02X}>" for value in character.encode("utf-8") + ] + # All-or-nothing per character: a single missing byte token sends + # the whole character down the unk path rather than emitting a + # partial expansion. + if all(token in self.vocab for token in byte_tokens): + # Deliberately leaves pending_unk alone. HuggingFace defers a + # pending unk past a successful byte expansion, so with + # <0xE4> missing, '中é' gives ['<0xC3>', '<0xA9>', '']. + # Probably an upstream quirk, but byte-parity is the point of + # this package, so it is reproduced rather than corrected. + symbols.extend(byte_tokens) + continue + if self.unk_token is None: + continue + if pending_unk and not self.fuse_unk: + symbols.append(self._unk_token_or_raise()) + pending_unk = True + if pending_unk: + symbols.append(self._unk_token_or_raise()) + return symbols + + def _unk_token_or_raise(self) -> str: + """Return the unk token, or fail if the vocabulary has no id for it.""" + if self.unk_token not in self.vocab: + raise ModelError( + f"unk token {self.unk_token!r} is not in the vocabulary, so " + "out-of-vocabulary input cannot be represented" + ) + return self.unk_token + + # -- core merge logic (lifted from ByteLevelBPE._apply_bpe) ------------ + def _apply_merges(self, symbols: Sequence[str]) -> list[str]: + """Apply BPE merges to already-resolved symbols.""" + symbols = tuple(symbols) if len(symbols) < 2: return list(symbols) while True: diff --git a/src/ubi_tokenizer/trainers/bpe.py b/src/ubi_tokenizer/trainers/bpe.py index 3fc3fa6..a021631 100644 --- a/src/ubi_tokenizer/trainers/bpe.py +++ b/src/ubi_tokenizer/trainers/bpe.py @@ -14,6 +14,7 @@ from collections.abc import Iterable, Iterator from typing import Optional +from ubi_tokenizer._validation import reject_unimplemented_affixes from ubi_tokenizer.byte_level import BYTE_ENCODER, encode_token_bytes from ubi_tokenizer.models.bpe import BPE from ubi_tokenizer.pre_tokenizers.byte_level import ByteLevel @@ -40,6 +41,7 @@ def __init__( continuing_subword_prefix: Optional[str] = None, end_of_word_suffix: Optional[str] = None, ) -> None: + reject_unimplemented_affixes(continuing_subword_prefix, end_of_word_suffix) self.vocab_size = vocab_size self.min_frequency = min_frequency self.special_tokens = list(special_tokens or []) diff --git a/tests/fixtures/hf_bpe_flags.json b/tests/fixtures/hf_bpe_flags.json new file mode 100644 index 0000000..678b13b --- /dev/null +++ b/tests/fixtures/hf_bpe_flags.json @@ -0,0 +1,1691 @@ +{ + "_generated_by": "scripts/gen_hf_flag_fixtures.py", + "_ground_truth": "huggingface tokenizers 0.23.1", + "_warning": "Do not hand-edit. Regenerate with the pinned tokenizers version; some recorded behaviour is version-sensitive.", + "cases": [ + { + "name": "baseline_no_flags", + "why": "merge loop must not regress", + "vocab": { + "a": 0, + "b": 1, + "c": 2, + "ab": 3, + "abc": 4 + }, + "merges": [ + [ + "a", + "b" + ], + [ + "ab", + "c" + ] + ], + "flags": {}, + "expected": { + "abc": [ + "abc" + ], + "ab": [ + "ab" + ], + "a": [ + "a" + ], + "abz": [ + "ab" + ], + "zz": [], + "az": [ + "a" + ], + "": [] + } + }, + { + "name": "ignore_merges_off_control", + "why": "control for the case below; proves the flag is what changes output", + "vocab": { + "x": 0, + "y": 1, + "xy": 2, + "z": 3, + "xz": 4 + }, + "merges": [ + [ + "x", + "z" + ] + ], + "flags": {}, + "expected": { + "xy": [ + "x", + "y" + ], + "xz": [ + "xz" + ], + "xyz": [ + "x", + "y", + "z" + ], + "xyxy": [ + "x", + "y", + "x", + "y" + ], + "": [] + } + }, + { + "name": "ignore_merges_discriminating", + "why": "'xy' is in vocab but no merge reaches it, so the flag alone decides", + "vocab": { + "x": 0, + "y": 1, + "xy": 2, + "z": 3, + "xz": 4 + }, + "merges": [ + [ + "x", + "z" + ] + ], + "flags": { + "ignore_merges": true + }, + "expected": { + "xy": [ + "xy" + ], + "xz": [ + "xz" + ], + "xyz": [ + "x", + "y", + "z" + ], + "xyxy": [ + "x", + "y", + "x", + "y" + ], + "": [] + } + }, + { + "name": "drop_when_no_unk", + "why": "out-of-vocab characters are dropped, not turned into an invalid id", + "vocab": { + "a": 0, + "b": 1 + }, + "merges": [], + "flags": {}, + "expected": { + "zz": [], + "azb": [ + "a", + "b" + ], + "az": [ + "a" + ], + "za": [ + "a" + ], + "中": [] + } + }, + { + "name": "unk_only", + "why": "one unk per out-of-vocab character", + "vocab": { + "a": 0, + "b": 1, + "ab": 2, + "": 3 + }, + "merges": [ + [ + "a", + "b" + ] + ], + "flags": { + "unk_token": "" + }, + "expected": { + "ab": [ + "ab" + ], + "az": [ + "a", + "" + ], + "zz": [ + "", + "" + ], + "az z": [ + "a", + "", + "", + "" + ], + "中": [ + "" + ], + "": [ + "", + "", + "", + "", + "" + ] + } + }, + { + "name": "unk_fused", + "why": "consecutive pending unks coalesce", + "vocab": { + "a": 0, + "b": 1, + "ab": 2, + "": 3 + }, + "merges": [ + [ + "a", + "b" + ] + ], + "flags": { + "unk_token": "", + "fuse_unk": true + }, + "expected": { + "ab": [ + "ab" + ], + "az": [ + "a", + "" + ], + "zz": [ + "" + ], + "az z": [ + "a", + "" + ], + "中": [ + "" + ], + "azza": [ + "a", + "", + "a" + ] + } + }, + { + "name": "unk_token_present_in_vocab", + "why": "an in-vocab unk string must not swallow a following genuine unk", + "vocab": { + "a": 0, + "U": 1 + }, + "merges": [], + "flags": { + "unk_token": "U", + "fuse_unk": true + }, + "expected": { + "Uz": [ + "U", + "U" + ], + "zU": [ + "U", + "U" + ], + "aUza": [ + "a", + "U", + "U", + "a" + ], + "UU": [ + "U", + "U" + ] + } + }, + { + "name": "byte_fallback_with_unk", + "why": "byte expansion, including bytes below 0x10", + "vocab": { + "a": 0, + "": 1, + "<0x00>": 2, + "<0x01>": 3, + "<0x02>": 4, + "<0x03>": 5, + "<0x04>": 6, + "<0x05>": 7, + "<0x06>": 8, + "<0x07>": 9, + "<0x08>": 10, + "<0x09>": 11, + "<0x0A>": 12, + "<0x0B>": 13, + "<0x0C>": 14, + "<0x0D>": 15, + "<0x0E>": 16, + "<0x0F>": 17, + "<0x10>": 18, + "<0x11>": 19, + "<0x12>": 20, + "<0x13>": 21, + "<0x14>": 22, + "<0x15>": 23, + "<0x16>": 24, + "<0x17>": 25, + "<0x18>": 26, + "<0x19>": 27, + "<0x1A>": 28, + "<0x1B>": 29, + "<0x1C>": 30, + "<0x1D>": 31, + "<0x1E>": 32, + "<0x1F>": 33, + "<0x20>": 34, + "<0x21>": 35, + "<0x22>": 36, + "<0x23>": 37, + "<0x24>": 38, + "<0x25>": 39, + "<0x26>": 40, + "<0x27>": 41, + "<0x28>": 42, + "<0x29>": 43, + "<0x2A>": 44, + "<0x2B>": 45, + "<0x2C>": 46, + "<0x2D>": 47, + "<0x2E>": 48, + "<0x2F>": 49, + "<0x30>": 50, + "<0x31>": 51, + "<0x32>": 52, + "<0x33>": 53, + "<0x34>": 54, + "<0x35>": 55, + "<0x36>": 56, + "<0x37>": 57, + "<0x38>": 58, + "<0x39>": 59, + "<0x3A>": 60, + "<0x3B>": 61, + "<0x3C>": 62, + "<0x3D>": 63, + "<0x3E>": 64, + "<0x3F>": 65, + "<0x40>": 66, + "<0x41>": 67, + "<0x42>": 68, + "<0x43>": 69, + "<0x44>": 70, + "<0x45>": 71, + "<0x46>": 72, + "<0x47>": 73, + "<0x48>": 74, + "<0x49>": 75, + "<0x4A>": 76, + "<0x4B>": 77, + "<0x4C>": 78, + "<0x4D>": 79, + "<0x4E>": 80, + "<0x4F>": 81, + "<0x50>": 82, + "<0x51>": 83, + "<0x52>": 84, + "<0x53>": 85, + "<0x54>": 86, + "<0x55>": 87, + "<0x56>": 88, + "<0x57>": 89, + "<0x58>": 90, + "<0x59>": 91, + "<0x5A>": 92, + "<0x5B>": 93, + "<0x5C>": 94, + "<0x5D>": 95, + "<0x5E>": 96, + "<0x5F>": 97, + "<0x60>": 98, + "<0x61>": 99, + "<0x62>": 100, + "<0x63>": 101, + "<0x64>": 102, + "<0x65>": 103, + "<0x66>": 104, + "<0x67>": 105, + "<0x68>": 106, + "<0x69>": 107, + "<0x6A>": 108, + "<0x6B>": 109, + "<0x6C>": 110, + "<0x6D>": 111, + "<0x6E>": 112, + "<0x6F>": 113, + "<0x70>": 114, + "<0x71>": 115, + "<0x72>": 116, + "<0x73>": 117, + "<0x74>": 118, + "<0x75>": 119, + "<0x76>": 120, + "<0x77>": 121, + "<0x78>": 122, + "<0x79>": 123, + "<0x7A>": 124, + "<0x7B>": 125, + "<0x7C>": 126, + "<0x7D>": 127, + "<0x7E>": 128, + "<0x7F>": 129, + "<0x80>": 130, + "<0x81>": 131, + "<0x82>": 132, + "<0x83>": 133, + "<0x84>": 134, + "<0x85>": 135, + "<0x86>": 136, + "<0x87>": 137, + "<0x88>": 138, + "<0x89>": 139, + "<0x8A>": 140, + "<0x8B>": 141, + "<0x8C>": 142, + "<0x8D>": 143, + "<0x8E>": 144, + "<0x8F>": 145, + "<0x90>": 146, + "<0x91>": 147, + "<0x92>": 148, + "<0x93>": 149, + "<0x94>": 150, + "<0x95>": 151, + "<0x96>": 152, + "<0x97>": 153, + "<0x98>": 154, + "<0x99>": 155, + "<0x9A>": 156, + "<0x9B>": 157, + "<0x9C>": 158, + "<0x9D>": 159, + "<0x9E>": 160, + "<0x9F>": 161, + "<0xA0>": 162, + "<0xA1>": 163, + "<0xA2>": 164, + "<0xA3>": 165, + "<0xA4>": 166, + "<0xA5>": 167, + "<0xA6>": 168, + "<0xA7>": 169, + "<0xA8>": 170, + "<0xA9>": 171, + "<0xAA>": 172, + "<0xAB>": 173, + "<0xAC>": 174, + "<0xAD>": 175, + "<0xAE>": 176, + "<0xAF>": 177, + "<0xB0>": 178, + "<0xB1>": 179, + "<0xB2>": 180, + "<0xB3>": 181, + "<0xB4>": 182, + "<0xB5>": 183, + "<0xB6>": 184, + "<0xB7>": 185, + "<0xB8>": 186, + "<0xB9>": 187, + "<0xBA>": 188, + "<0xBB>": 189, + "<0xBC>": 190, + "<0xBD>": 191, + "<0xBE>": 192, + "<0xBF>": 193, + "<0xC0>": 194, + "<0xC1>": 195, + "<0xC2>": 196, + "<0xC3>": 197, + "<0xC4>": 198, + "<0xC5>": 199, + "<0xC6>": 200, + "<0xC7>": 201, + "<0xC8>": 202, + "<0xC9>": 203, + "<0xCA>": 204, + "<0xCB>": 205, + "<0xCC>": 206, + "<0xCD>": 207, + "<0xCE>": 208, + "<0xCF>": 209, + "<0xD0>": 210, + "<0xD1>": 211, + "<0xD2>": 212, + "<0xD3>": 213, + "<0xD4>": 214, + "<0xD5>": 215, + "<0xD6>": 216, + "<0xD7>": 217, + "<0xD8>": 218, + "<0xD9>": 219, + "<0xDA>": 220, + "<0xDB>": 221, + "<0xDC>": 222, + "<0xDD>": 223, + "<0xDE>": 224, + "<0xDF>": 225, + "<0xE0>": 226, + "<0xE1>": 227, + "<0xE2>": 228, + "<0xE3>": 229, + "<0xE4>": 230, + "<0xE5>": 231, + "<0xE6>": 232, + "<0xE7>": 233, + "<0xE8>": 234, + "<0xE9>": 235, + "<0xEA>": 236, + "<0xEB>": 237, + "<0xEC>": 238, + "<0xED>": 239, + "<0xEE>": 240, + "<0xEF>": 241, + "<0xF0>": 242, + "<0xF1>": 243, + "<0xF2>": 244, + "<0xF3>": 245, + "<0xF4>": 246, + "<0xF5>": 247, + "<0xF6>": 248, + "<0xF7>": 249, + "<0xF8>": 250, + "<0xF9>": 251, + "<0xFA>": 252, + "<0xFB>": 253, + "<0xFC>": 254, + "<0xFD>": 255, + "<0xFE>": 256, + "<0xFF>": 257 + }, + "merges": [], + "flags": { + "unk_token": "", + "byte_fallback": true + }, + "expected": { + "a": [ + "a" + ], + "az": [ + "a", + "<0x7A>" + ], + "中": [ + "<0xE4>", + "<0xB8>", + "<0xAD>" + ], + "é": [ + "<0xC3>", + "<0xA9>" + ], + "abc": [ + "a", + "<0x62>", + "<0x63>" + ], + "€": [ + "<0xE2>", + "<0x82>", + "<0xAC>" + ], + "\t": [ + "<0x09>" + ], + "\n": [ + "<0x0A>" + ], + "\u0001": [ + "<0x01>" + ], + "\u000f": [ + "<0x0F>" + ] + } + }, + { + "name": "byte_fallback_without_unk", + "why": "no unk to fall back on; misses are dropped", + "vocab": { + "a": 0, + "<0x00>": 1, + "<0x01>": 2, + "<0x02>": 3, + "<0x03>": 4, + "<0x04>": 5, + "<0x05>": 6, + "<0x06>": 7, + "<0x07>": 8, + "<0x08>": 9, + "<0x09>": 10, + "<0x0A>": 11, + "<0x0B>": 12, + "<0x0C>": 13, + "<0x0D>": 14, + "<0x0E>": 15, + "<0x0F>": 16, + "<0x10>": 17, + "<0x11>": 18, + "<0x12>": 19, + "<0x13>": 20, + "<0x14>": 21, + "<0x15>": 22, + "<0x16>": 23, + "<0x17>": 24, + "<0x18>": 25, + "<0x19>": 26, + "<0x1A>": 27, + "<0x1B>": 28, + "<0x1C>": 29, + "<0x1D>": 30, + "<0x1E>": 31, + "<0x1F>": 32, + "<0x20>": 33, + "<0x21>": 34, + "<0x22>": 35, + "<0x23>": 36, + "<0x24>": 37, + "<0x25>": 38, + "<0x26>": 39, + "<0x27>": 40, + "<0x28>": 41, + "<0x29>": 42, + "<0x2A>": 43, + "<0x2B>": 44, + "<0x2C>": 45, + "<0x2D>": 46, + "<0x2E>": 47, + "<0x2F>": 48, + "<0x30>": 49, + "<0x31>": 50, + "<0x32>": 51, + "<0x33>": 52, + "<0x34>": 53, + "<0x35>": 54, + "<0x36>": 55, + "<0x37>": 56, + "<0x38>": 57, + "<0x39>": 58, + "<0x3A>": 59, + "<0x3B>": 60, + "<0x3C>": 61, + "<0x3D>": 62, + "<0x3E>": 63, + "<0x3F>": 64, + "<0x40>": 65, + "<0x41>": 66, + "<0x42>": 67, + "<0x43>": 68, + "<0x44>": 69, + "<0x45>": 70, + "<0x46>": 71, + "<0x47>": 72, + "<0x48>": 73, + "<0x49>": 74, + "<0x4A>": 75, + "<0x4B>": 76, + "<0x4C>": 77, + "<0x4D>": 78, + "<0x4E>": 79, + "<0x4F>": 80, + "<0x50>": 81, + "<0x51>": 82, + "<0x52>": 83, + "<0x53>": 84, + "<0x54>": 85, + "<0x55>": 86, + "<0x56>": 87, + "<0x57>": 88, + "<0x58>": 89, + "<0x59>": 90, + "<0x5A>": 91, + "<0x5B>": 92, + "<0x5C>": 93, + "<0x5D>": 94, + "<0x5E>": 95, + "<0x5F>": 96, + "<0x60>": 97, + "<0x61>": 98, + "<0x62>": 99, + "<0x63>": 100, + "<0x64>": 101, + "<0x65>": 102, + "<0x66>": 103, + "<0x67>": 104, + "<0x68>": 105, + "<0x69>": 106, + "<0x6A>": 107, + "<0x6B>": 108, + "<0x6C>": 109, + "<0x6D>": 110, + "<0x6E>": 111, + "<0x6F>": 112, + "<0x70>": 113, + "<0x71>": 114, + "<0x72>": 115, + "<0x73>": 116, + "<0x74>": 117, + "<0x75>": 118, + "<0x76>": 119, + "<0x77>": 120, + "<0x78>": 121, + "<0x79>": 122, + "<0x7A>": 123, + "<0x7B>": 124, + "<0x7C>": 125, + "<0x7D>": 126, + "<0x7E>": 127, + "<0x7F>": 128, + "<0x80>": 129, + "<0x81>": 130, + "<0x82>": 131, + "<0x83>": 132, + "<0x84>": 133, + "<0x85>": 134, + "<0x86>": 135, + "<0x87>": 136, + "<0x88>": 137, + "<0x89>": 138, + "<0x8A>": 139, + "<0x8B>": 140, + "<0x8C>": 141, + "<0x8D>": 142, + "<0x8E>": 143, + "<0x8F>": 144, + "<0x90>": 145, + "<0x91>": 146, + "<0x92>": 147, + "<0x93>": 148, + "<0x94>": 149, + "<0x95>": 150, + "<0x96>": 151, + "<0x97>": 152, + "<0x98>": 153, + "<0x99>": 154, + "<0x9A>": 155, + "<0x9B>": 156, + "<0x9C>": 157, + "<0x9D>": 158, + "<0x9E>": 159, + "<0x9F>": 160, + "<0xA0>": 161, + "<0xA1>": 162, + "<0xA2>": 163, + "<0xA3>": 164, + "<0xA4>": 165, + "<0xA5>": 166, + "<0xA6>": 167, + "<0xA7>": 168, + "<0xA8>": 169, + "<0xA9>": 170, + "<0xAA>": 171, + "<0xAB>": 172, + "<0xAC>": 173, + "<0xAE>": 174, + "<0xAF>": 175, + "<0xB0>": 176, + "<0xB1>": 177, + "<0xB2>": 178, + "<0xB3>": 179, + "<0xB4>": 180, + "<0xB5>": 181, + "<0xB6>": 182, + "<0xB7>": 183, + "<0xB9>": 184, + "<0xBA>": 185, + "<0xBB>": 186, + "<0xBC>": 187, + "<0xBD>": 188, + "<0xBE>": 189, + "<0xBF>": 190, + "<0xC0>": 191, + "<0xC1>": 192, + "<0xC2>": 193, + "<0xC3>": 194, + "<0xC4>": 195, + "<0xC5>": 196, + "<0xC6>": 197, + "<0xC7>": 198, + "<0xC8>": 199, + "<0xC9>": 200, + "<0xCA>": 201, + "<0xCB>": 202, + "<0xCC>": 203, + "<0xCD>": 204, + "<0xCE>": 205, + "<0xCF>": 206, + "<0xD0>": 207, + "<0xD1>": 208, + "<0xD2>": 209, + "<0xD3>": 210, + "<0xD4>": 211, + "<0xD5>": 212, + "<0xD6>": 213, + "<0xD7>": 214, + "<0xD8>": 215, + "<0xD9>": 216, + "<0xDA>": 217, + "<0xDB>": 218, + "<0xDC>": 219, + "<0xDD>": 220, + "<0xDE>": 221, + "<0xDF>": 222, + "<0xE0>": 223, + "<0xE1>": 224, + "<0xE2>": 225, + "<0xE3>": 226, + "<0xE5>": 227, + "<0xE6>": 228, + "<0xE7>": 229, + "<0xE8>": 230, + "<0xE9>": 231, + "<0xEA>": 232, + "<0xEB>": 233, + "<0xEC>": 234, + "<0xED>": 235, + "<0xEE>": 236, + "<0xEF>": 237, + "<0xF0>": 238, + "<0xF1>": 239, + "<0xF2>": 240, + "<0xF3>": 241, + "<0xF4>": 242, + "<0xF5>": 243, + "<0xF6>": 244, + "<0xF7>": 245, + "<0xF8>": 246, + "<0xF9>": 247, + "<0xFA>": 248, + "<0xFB>": 249, + "<0xFC>": 250, + "<0xFD>": 251, + "<0xFE>": 252, + "<0xFF>": 253 + }, + "merges": [], + "flags": { + "byte_fallback": true + }, + "expected": { + "a": [ + "a" + ], + "az": [ + "a", + "<0x7A>" + ], + "中": [], + "é": [ + "<0xC3>", + "<0xA9>" + ], + "a中a": [ + "a", + "a" + ], + "\t": [ + "<0x09>" + ], + "\n": [ + "<0x0A>" + ], + "\u0001": [ + "<0x01>" + ], + "\u000f": [ + "<0x0F>" + ] + } + }, + { + "name": "bare_ascii_absent_but_byte_token_present", + "why": "byte fallback also fires for plain ASCII when the bare char is missing", + "vocab": { + "": 0, + "<0x61>": 1, + "<0x62>": 2, + "<0x63>": 3 + }, + "merges": [], + "flags": { + "unk_token": "", + "byte_fallback": true + }, + "expected": { + "abc": [ + "<0x61>", + "<0x62>", + "<0x63>" + ], + "a": [ + "<0x61>" + ], + "abd": [ + "<0x61>", + "<0x62>", + "" + ] + } + }, + { + "name": "bf_partial_bytes", + "why": "all-or-nothing per character, and the pending-unk slot is NOT flushed by a byte-fallback hit", + "vocab": { + "a": 0, + "": 1, + "<0x00>": 2, + "<0x01>": 3, + "<0x02>": 4, + "<0x03>": 5, + "<0x04>": 6, + "<0x05>": 7, + "<0x06>": 8, + "<0x07>": 9, + "<0x08>": 10, + "<0x09>": 11, + "<0x0A>": 12, + "<0x0B>": 13, + "<0x0C>": 14, + "<0x0D>": 15, + "<0x0E>": 16, + "<0x0F>": 17, + "<0x10>": 18, + "<0x11>": 19, + "<0x12>": 20, + "<0x13>": 21, + "<0x14>": 22, + "<0x15>": 23, + "<0x16>": 24, + "<0x17>": 25, + "<0x18>": 26, + "<0x19>": 27, + "<0x1A>": 28, + "<0x1B>": 29, + "<0x1C>": 30, + "<0x1D>": 31, + "<0x1E>": 32, + "<0x1F>": 33, + "<0x20>": 34, + "<0x21>": 35, + "<0x22>": 36, + "<0x23>": 37, + "<0x24>": 38, + "<0x25>": 39, + "<0x26>": 40, + "<0x27>": 41, + "<0x28>": 42, + "<0x29>": 43, + "<0x2A>": 44, + "<0x2B>": 45, + "<0x2C>": 46, + "<0x2D>": 47, + "<0x2E>": 48, + "<0x2F>": 49, + "<0x30>": 50, + "<0x31>": 51, + "<0x32>": 52, + "<0x33>": 53, + "<0x34>": 54, + "<0x35>": 55, + "<0x36>": 56, + "<0x37>": 57, + "<0x38>": 58, + "<0x39>": 59, + "<0x3A>": 60, + "<0x3B>": 61, + "<0x3C>": 62, + "<0x3D>": 63, + "<0x3E>": 64, + "<0x3F>": 65, + "<0x40>": 66, + "<0x41>": 67, + "<0x42>": 68, + "<0x43>": 69, + "<0x44>": 70, + "<0x45>": 71, + "<0x46>": 72, + "<0x47>": 73, + "<0x48>": 74, + "<0x49>": 75, + "<0x4A>": 76, + "<0x4B>": 77, + "<0x4C>": 78, + "<0x4D>": 79, + "<0x4E>": 80, + "<0x4F>": 81, + "<0x50>": 82, + "<0x51>": 83, + "<0x52>": 84, + "<0x53>": 85, + "<0x54>": 86, + "<0x55>": 87, + "<0x56>": 88, + "<0x57>": 89, + "<0x58>": 90, + "<0x59>": 91, + "<0x5A>": 92, + "<0x5B>": 93, + "<0x5C>": 94, + "<0x5D>": 95, + "<0x5E>": 96, + "<0x5F>": 97, + "<0x60>": 98, + "<0x61>": 99, + "<0x62>": 100, + "<0x63>": 101, + "<0x64>": 102, + "<0x65>": 103, + "<0x66>": 104, + "<0x67>": 105, + "<0x68>": 106, + "<0x69>": 107, + "<0x6A>": 108, + "<0x6B>": 109, + "<0x6C>": 110, + "<0x6D>": 111, + "<0x6E>": 112, + "<0x6F>": 113, + "<0x70>": 114, + "<0x71>": 115, + "<0x72>": 116, + "<0x73>": 117, + "<0x74>": 118, + "<0x75>": 119, + "<0x76>": 120, + "<0x77>": 121, + "<0x78>": 122, + "<0x79>": 123, + "<0x7A>": 124, + "<0x7B>": 125, + "<0x7C>": 126, + "<0x7D>": 127, + "<0x7E>": 128, + "<0x7F>": 129, + "<0x80>": 130, + "<0x81>": 131, + "<0x82>": 132, + "<0x83>": 133, + "<0x84>": 134, + "<0x85>": 135, + "<0x86>": 136, + "<0x87>": 137, + "<0x88>": 138, + "<0x89>": 139, + "<0x8A>": 140, + "<0x8B>": 141, + "<0x8C>": 142, + "<0x8D>": 143, + "<0x8E>": 144, + "<0x8F>": 145, + "<0x90>": 146, + "<0x91>": 147, + "<0x92>": 148, + "<0x93>": 149, + "<0x94>": 150, + "<0x95>": 151, + "<0x96>": 152, + "<0x97>": 153, + "<0x98>": 154, + "<0x99>": 155, + "<0x9A>": 156, + "<0x9B>": 157, + "<0x9C>": 158, + "<0x9D>": 159, + "<0x9E>": 160, + "<0x9F>": 161, + "<0xA0>": 162, + "<0xA1>": 163, + "<0xA2>": 164, + "<0xA3>": 165, + "<0xA4>": 166, + "<0xA5>": 167, + "<0xA6>": 168, + "<0xA7>": 169, + "<0xA8>": 170, + "<0xA9>": 171, + "<0xAA>": 172, + "<0xAB>": 173, + "<0xAC>": 174, + "<0xAD>": 175, + "<0xAE>": 176, + "<0xAF>": 177, + "<0xB0>": 178, + "<0xB1>": 179, + "<0xB2>": 180, + "<0xB3>": 181, + "<0xB4>": 182, + "<0xB5>": 183, + "<0xB6>": 184, + "<0xB7>": 185, + "<0xB8>": 186, + "<0xB9>": 187, + "<0xBA>": 188, + "<0xBB>": 189, + "<0xBC>": 190, + "<0xBD>": 191, + "<0xBE>": 192, + "<0xBF>": 193, + "<0xC0>": 194, + "<0xC1>": 195, + "<0xC2>": 196, + "<0xC3>": 197, + "<0xC4>": 198, + "<0xC5>": 199, + "<0xC6>": 200, + "<0xC7>": 201, + "<0xC8>": 202, + "<0xC9>": 203, + "<0xCA>": 204, + "<0xCB>": 205, + "<0xCC>": 206, + "<0xCD>": 207, + "<0xCE>": 208, + "<0xCF>": 209, + "<0xD0>": 210, + "<0xD1>": 211, + "<0xD2>": 212, + "<0xD3>": 213, + "<0xD4>": 214, + "<0xD5>": 215, + "<0xD6>": 216, + "<0xD7>": 217, + "<0xD8>": 218, + "<0xD9>": 219, + "<0xDA>": 220, + "<0xDB>": 221, + "<0xDC>": 222, + "<0xDD>": 223, + "<0xDE>": 224, + "<0xDF>": 225, + "<0xE0>": 226, + "<0xE1>": 227, + "<0xE2>": 228, + "<0xE3>": 229, + "<0xE5>": 230, + "<0xE6>": 231, + "<0xE7>": 232, + "<0xE8>": 233, + "<0xE9>": 234, + "<0xEA>": 235, + "<0xEB>": 236, + "<0xEC>": 237, + "<0xED>": 238, + "<0xEE>": 239, + "<0xEF>": 240, + "<0xF0>": 241, + "<0xF1>": 242, + "<0xF2>": 243, + "<0xF3>": 244, + "<0xF4>": 245, + "<0xF5>": 246, + "<0xF6>": 247, + "<0xF7>": 248, + "<0xF8>": 249, + "<0xF9>": 250, + "<0xFA>": 251, + "<0xFB>": 252, + "<0xFC>": 253, + "<0xFD>": 254, + "<0xFE>": 255, + "<0xFF>": 256 + }, + "merges": [], + "flags": { + "unk_token": "", + "byte_fallback": true + }, + "expected": { + "中": [ + "" + ], + "中é": [ + "<0xC3>", + "<0xA9>", + "" + ], + "é中": [ + "<0xC3>", + "<0xA9>", + "" + ], + "中中é": [ + "", + "<0xC3>", + "<0xA9>", + "" + ], + "中é中": [ + "<0xC3>", + "<0xA9>", + "", + "" + ], + "中中": [ + "", + "" + ], + "aé中": [ + "a", + "<0xC3>", + "<0xA9>", + "" + ], + "中a": [ + "", + "a" + ] + } + }, + { + "name": "bf_partial_bytes_fused", + "why": "same slot behaviour with coalescing on top", + "vocab": { + "a": 0, + "": 1, + "<0x00>": 2, + "<0x01>": 3, + "<0x02>": 4, + "<0x03>": 5, + "<0x04>": 6, + "<0x05>": 7, + "<0x06>": 8, + "<0x07>": 9, + "<0x08>": 10, + "<0x09>": 11, + "<0x0A>": 12, + "<0x0B>": 13, + "<0x0C>": 14, + "<0x0D>": 15, + "<0x0E>": 16, + "<0x0F>": 17, + "<0x10>": 18, + "<0x11>": 19, + "<0x12>": 20, + "<0x13>": 21, + "<0x14>": 22, + "<0x15>": 23, + "<0x16>": 24, + "<0x17>": 25, + "<0x18>": 26, + "<0x19>": 27, + "<0x1A>": 28, + "<0x1B>": 29, + "<0x1C>": 30, + "<0x1D>": 31, + "<0x1E>": 32, + "<0x1F>": 33, + "<0x20>": 34, + "<0x21>": 35, + "<0x22>": 36, + "<0x23>": 37, + "<0x24>": 38, + "<0x25>": 39, + "<0x26>": 40, + "<0x27>": 41, + "<0x28>": 42, + "<0x29>": 43, + "<0x2A>": 44, + "<0x2B>": 45, + "<0x2C>": 46, + "<0x2D>": 47, + "<0x2E>": 48, + "<0x2F>": 49, + "<0x30>": 50, + "<0x31>": 51, + "<0x32>": 52, + "<0x33>": 53, + "<0x34>": 54, + "<0x35>": 55, + "<0x36>": 56, + "<0x37>": 57, + "<0x38>": 58, + "<0x39>": 59, + "<0x3A>": 60, + "<0x3B>": 61, + "<0x3C>": 62, + "<0x3D>": 63, + "<0x3E>": 64, + "<0x3F>": 65, + "<0x40>": 66, + "<0x41>": 67, + "<0x42>": 68, + "<0x43>": 69, + "<0x44>": 70, + "<0x45>": 71, + "<0x46>": 72, + "<0x47>": 73, + "<0x48>": 74, + "<0x49>": 75, + "<0x4A>": 76, + "<0x4B>": 77, + "<0x4C>": 78, + "<0x4D>": 79, + "<0x4E>": 80, + "<0x4F>": 81, + "<0x50>": 82, + "<0x51>": 83, + "<0x52>": 84, + "<0x53>": 85, + "<0x54>": 86, + "<0x55>": 87, + "<0x56>": 88, + "<0x57>": 89, + "<0x58>": 90, + "<0x59>": 91, + "<0x5A>": 92, + "<0x5B>": 93, + "<0x5C>": 94, + "<0x5D>": 95, + "<0x5E>": 96, + "<0x5F>": 97, + "<0x60>": 98, + "<0x61>": 99, + "<0x62>": 100, + "<0x63>": 101, + "<0x64>": 102, + "<0x65>": 103, + "<0x66>": 104, + "<0x67>": 105, + "<0x68>": 106, + "<0x69>": 107, + "<0x6A>": 108, + "<0x6B>": 109, + "<0x6C>": 110, + "<0x6D>": 111, + "<0x6E>": 112, + "<0x6F>": 113, + "<0x70>": 114, + "<0x71>": 115, + "<0x72>": 116, + "<0x73>": 117, + "<0x74>": 118, + "<0x75>": 119, + "<0x76>": 120, + "<0x77>": 121, + "<0x78>": 122, + "<0x79>": 123, + "<0x7A>": 124, + "<0x7B>": 125, + "<0x7C>": 126, + "<0x7D>": 127, + "<0x7E>": 128, + "<0x7F>": 129, + "<0x80>": 130, + "<0x81>": 131, + "<0x82>": 132, + "<0x83>": 133, + "<0x84>": 134, + "<0x85>": 135, + "<0x86>": 136, + "<0x87>": 137, + "<0x88>": 138, + "<0x89>": 139, + "<0x8A>": 140, + "<0x8B>": 141, + "<0x8C>": 142, + "<0x8D>": 143, + "<0x8E>": 144, + "<0x8F>": 145, + "<0x90>": 146, + "<0x91>": 147, + "<0x92>": 148, + "<0x93>": 149, + "<0x94>": 150, + "<0x95>": 151, + "<0x96>": 152, + "<0x97>": 153, + "<0x98>": 154, + "<0x99>": 155, + "<0x9A>": 156, + "<0x9B>": 157, + "<0x9C>": 158, + "<0x9D>": 159, + "<0x9E>": 160, + "<0x9F>": 161, + "<0xA0>": 162, + "<0xA1>": 163, + "<0xA2>": 164, + "<0xA3>": 165, + "<0xA4>": 166, + "<0xA5>": 167, + "<0xA6>": 168, + "<0xA7>": 169, + "<0xA8>": 170, + "<0xA9>": 171, + "<0xAA>": 172, + "<0xAB>": 173, + "<0xAC>": 174, + "<0xAD>": 175, + "<0xAE>": 176, + "<0xAF>": 177, + "<0xB0>": 178, + "<0xB1>": 179, + "<0xB2>": 180, + "<0xB3>": 181, + "<0xB4>": 182, + "<0xB5>": 183, + "<0xB6>": 184, + "<0xB7>": 185, + "<0xB8>": 186, + "<0xB9>": 187, + "<0xBA>": 188, + "<0xBB>": 189, + "<0xBC>": 190, + "<0xBD>": 191, + "<0xBE>": 192, + "<0xBF>": 193, + "<0xC0>": 194, + "<0xC1>": 195, + "<0xC2>": 196, + "<0xC3>": 197, + "<0xC4>": 198, + "<0xC5>": 199, + "<0xC6>": 200, + "<0xC7>": 201, + "<0xC8>": 202, + "<0xC9>": 203, + "<0xCA>": 204, + "<0xCB>": 205, + "<0xCC>": 206, + "<0xCD>": 207, + "<0xCE>": 208, + "<0xCF>": 209, + "<0xD0>": 210, + "<0xD1>": 211, + "<0xD2>": 212, + "<0xD3>": 213, + "<0xD4>": 214, + "<0xD5>": 215, + "<0xD6>": 216, + "<0xD7>": 217, + "<0xD8>": 218, + "<0xD9>": 219, + "<0xDA>": 220, + "<0xDB>": 221, + "<0xDC>": 222, + "<0xDD>": 223, + "<0xDE>": 224, + "<0xDF>": 225, + "<0xE0>": 226, + "<0xE1>": 227, + "<0xE2>": 228, + "<0xE3>": 229, + "<0xE5>": 230, + "<0xE6>": 231, + "<0xE7>": 232, + "<0xE8>": 233, + "<0xE9>": 234, + "<0xEA>": 235, + "<0xEB>": 236, + "<0xEC>": 237, + "<0xED>": 238, + "<0xEE>": 239, + "<0xEF>": 240, + "<0xF0>": 241, + "<0xF1>": 242, + "<0xF2>": 243, + "<0xF3>": 244, + "<0xF4>": 245, + "<0xF5>": 246, + "<0xF6>": 247, + "<0xF7>": 248, + "<0xF8>": 249, + "<0xF9>": 250, + "<0xFA>": 251, + "<0xFB>": 252, + "<0xFC>": 253, + "<0xFD>": 254, + "<0xFE>": 255, + "<0xFF>": 256 + }, + "merges": [], + "flags": { + "unk_token": "", + "byte_fallback": true, + "fuse_unk": true + }, + "expected": { + "中": [ + "" + ], + "中é": [ + "<0xC3>", + "<0xA9>", + "" + ], + "é中": [ + "<0xC3>", + "<0xA9>", + "" + ], + "中中é": [ + "<0xC3>", + "<0xA9>", + "" + ], + "中é中": [ + "<0xC3>", + "<0xA9>", + "" + ], + "中中": [ + "" + ], + "aé中": [ + "a", + "<0xC3>", + "<0xA9>", + "" + ], + "中a": [ + "", + "a" + ] + } + }, + { + "name": "merge_over_byte_tokens", + "why": "byte-fallback tokens participate in merges, so resolution precedes merging", + "vocab": { + "a": 0, + "": 1, + "<0xE4>": 2, + "<0xB8>": 3, + "<0xAD>": 4, + "<0xE4><0xB8>": 5, + "a<0xE4>": 6 + }, + "merges": [ + [ + "<0xE4>", + "<0xB8>" + ] + ], + "flags": { + "unk_token": "", + "byte_fallback": true + }, + "expected": { + "中": [ + "<0xE4><0xB8>", + "<0xAD>" + ], + "a中": [ + "a", + "<0xE4><0xB8>", + "<0xAD>" + ] + } + }, + { + "name": "merge_over_byte_tokens_rank_order", + "why": "merge rank over byte tokens changes the result", + "vocab": { + "a": 0, + "": 1, + "<0xE4>": 2, + "<0xB8>": 3, + "<0xAD>": 4, + "<0xE4><0xB8>": 5, + "a<0xE4>": 6 + }, + "merges": [ + [ + "a", + "<0xE4>" + ], + [ + "<0xE4>", + "<0xB8>" + ] + ], + "flags": { + "unk_token": "", + "byte_fallback": true + }, + "expected": { + "中": [ + "<0xE4><0xB8>", + "<0xAD>" + ], + "a中": [ + "a<0xE4>", + "<0xB8>", + "<0xAD>" + ] + } + }, + { + "name": "merge_over_unk_tokens", + "why": "generated unk tokens participate in merges too", + "vocab": { + "a": 0, + "": 1, + "": 2 + }, + "merges": [ + [ + "", + "" + ] + ], + "flags": { + "unk_token": "" + }, + "expected": { + "zz": [ + "" + ], + "azza": [ + "a", + "", + "a" + ], + "z": [ + "" + ], + "zzz": [ + "", + "" + ] + } + }, + { + "name": "fuse_unk_without_unk_token", + "why": "fuse_unk is inert with no unk token to fuse", + "vocab": { + "a": 0, + "b": 1 + }, + "merges": [], + "flags": { + "fuse_unk": true + }, + "expected": { + "azza": [ + "a", + "a" + ], + "zz": [], + "ab": [ + "a", + "b" + ] + } + }, + { + "name": "ignore_merges_falls_through_to_unk", + "why": "a piece absent from vocab still takes the normal resolve path", + "vocab": { + "a": 0, + "b": 1, + "ab": 2, + "": 3 + }, + "merges": [ + [ + "a", + "b" + ] + ], + "flags": { + "ignore_merges": true, + "unk_token": "" + }, + "expected": { + "ab": [ + "ab" + ], + "abz": [ + "ab", + "" + ], + "zz": [ + "", + "" + ], + "a": [ + "a" + ] + } + } + ] +} diff --git a/tests/test_bpe_flags.py b/tests/test_bpe_flags.py new file mode 100644 index 0000000..3e29c4e --- /dev/null +++ b/tests/test_bpe_flags.py @@ -0,0 +1,91 @@ +"""BPE option behaviour, checked against recorded HuggingFace output. + +The expectations in ``fixtures/hf_bpe_flags.json`` were measured by running real +``huggingface/tokenizers``; see ``scripts/gen_hf_flag_fixtures.py``. This module +reads that recording using only the standard library, so the suite keeps working +without any third-party package installed. +""" + +import json +from pathlib import Path + +import pytest + +from ubi_tokenizer.errors import ModelError +from ubi_tokenizer.models.bpe import BPE + +FIXTURE = Path(__file__).parent / "fixtures" / "hf_bpe_flags.json" + + +def _load_cases(): + # A missing fixture is a hard failure, never a skip: silently skipping would + # remove all HuggingFace parity coverage without anyone noticing. + if not FIXTURE.exists(): + raise AssertionError( + f"missing ground-truth fixture {FIXTURE}; " + "regenerate with scripts/gen_hf_flag_fixtures.py" + ) + return json.loads(FIXTURE.read_text(encoding="utf-8"))["cases"] + + +CASES = _load_cases() + + +def _model(case): + return BPE( + vocab=dict(case["vocab"]), + merges=[tuple(pair) for pair in case["merges"]], + **case["flags"], + ) + + +def _expectations(): + for case in CASES: + for text, expected in case["expected"].items(): + yield pytest.param(case, text, expected, id=f"{case['name']}-{text!r}") + + +@pytest.mark.parametrize(("case", "text", "expected"), list(_expectations())) +def test_matches_huggingface(case, text, expected): + assert _model(case).tokenize(text) == expected, case["why"] + + +def test_fixture_records_its_ground_truth_version(): + payload = json.loads(FIXTURE.read_text(encoding="utf-8")) + assert payload["_ground_truth"].startswith("huggingface tokenizers ") + + +def test_ignore_merges_is_actually_discriminating(): + # Guards the fixture set itself: if these two cases ever agree, the + # ignore_merges coverage has quietly become vacuous. + off = next(c for c in CASES if c["name"] == "ignore_merges_off_control") + on = next(c for c in CASES if c["name"] == "ignore_merges_discriminating") + assert off["expected"] != on["expected"] + + +# -- behaviour with no HuggingFace counterpart ------------------------------ + + +def test_unk_token_absent_from_vocab_raises(): + # HuggingFace raises lazily at tokenize time rather than at construction, + # and so do we. Constructing must stay quiet. + model = BPE(vocab={"a": 0}, merges=[], unk_token="") + assert model.tokenize("a") == ["a"] + with pytest.raises(ModelError, match=""): + model.tokenize("az") + + +def test_byte_fallback_hit_still_works_when_unk_is_absent_from_vocab(): + # Only the unk path needs the unk token, so a pure byte-fallback hit must + # not be dragged down by a missing unk. + vocab = {f"<0x{value:02X}>": value for value in range(256)} + model = BPE(vocab=vocab, merges=[], unk_token="", byte_fallback=True) + assert model.tokenize("a") == ["<0x61>"] + + +def test_lone_surrogate_propagates_encoding_error(): + # Pure Python can be handed a lone surrogate where HuggingFace's binding + # layer would already have rejected it. Propagating is deliberate. + model = BPE(vocab={"a": 0}, merges=[], byte_fallback=True) + with pytest.raises(UnicodeEncodeError): + model.tokenize("\ud800") diff --git a/tests/test_unsupported_flags.py b/tests/test_unsupported_flags.py new file mode 100644 index 0000000..89879d7 --- /dev/null +++ b/tests/test_unsupported_flags.py @@ -0,0 +1,123 @@ +"""Unimplemented BPE / trainer options must fail loudly instead of silently. + +``BPE`` and ``BpeTrainer`` accept several HuggingFace-compatible options that are +stored and serialized but never acted on. Accepting them silently means a caller +gets wrong tokenization with no signal, so a non-default value now raises. +Default values stay silent: they are true no-ops in HuggingFace too, and +GPT-2-lineage ``tokenizer.json`` files ship ``""`` for the affix options. +""" + +import pytest + +from ubi_tokenizer.errors import ModelError, UbiTokenizerError, UnsupportedFeatureError +from ubi_tokenizer.models.bpe import BPE +from ubi_tokenizer.trainers.bpe import BpeTrainer + + +def test_unsupported_feature_error_is_a_ubi_tokenizer_error(): + assert issubclass(UnsupportedFeatureError, UbiTokenizerError) + + +# -- dropout --------------------------------------------------------------- + + +@pytest.mark.parametrize("value", [None, 0.0]) +def test_no_op_dropout_is_accepted(value): + # HuggingFace treats both as "no dropout", so they must not raise. + assert BPE(dropout=value).dropout == value + + +@pytest.mark.parametrize("value", [0.1, 0.5, 1.0]) +def test_active_dropout_is_rejected(value): + with pytest.raises(UnsupportedFeatureError, match="dropout"): + BPE(dropout=value) + + +@pytest.mark.parametrize("value", [-0.1, 1.5]) +def test_out_of_range_dropout_is_rejected_as_invalid_model_state(value): + # HuggingFace rejects these at construction as invalid rather than + # unimplemented, so they are ModelError and not UnsupportedFeatureError. + with pytest.raises(ModelError, match="dropout"): + BPE(dropout=value) + + +# -- affix options -------------------------------------------------------- + + +@pytest.mark.parametrize("option", ["continuing_subword_prefix", "end_of_word_suffix"]) +@pytest.mark.parametrize("value", [None, ""]) +def test_no_op_affix_options_are_accepted(option, value): + assert getattr(BPE(**{option: value}), option) == value + + +@pytest.mark.parametrize( + ("option", "value"), + [("continuing_subword_prefix", "##"), ("end_of_word_suffix", "")], +) +def test_active_affix_options_are_rejected(option, value): + with pytest.raises(UnsupportedFeatureError, match=option): + BPE(**{option: value}) + + +@pytest.mark.parametrize( + ("option", "value"), + [("continuing_subword_prefix", "##"), ("end_of_word_suffix", "")], +) +def test_trainer_rejects_active_affix_options(option, value): + # The trainer stores these two and never reads them either; fixing only the + # model would leave half of the same defect in place. + with pytest.raises(UnsupportedFeatureError, match=option): + BpeTrainer(**{option: value}) + + +@pytest.mark.parametrize("option", ["continuing_subword_prefix", "end_of_word_suffix"]) +@pytest.mark.parametrize("value", [None, ""]) +def test_trainer_accepts_no_op_affix_options(option, value): + assert getattr(BpeTrainer(**{option: value}), option) == value + + +def test_trainer_defaults_are_accepted(): + trainer = BpeTrainer(vocab_size=64, min_frequency=1) + assert trainer.continuing_subword_prefix is None + assert trainer.end_of_word_suffix is None + + +# -- message quality ------------------------------------------------------- + + +def test_rejection_message_names_the_option_and_says_it_is_unimplemented(): + with pytest.raises(UnsupportedFeatureError) as excinfo: + BPE(end_of_word_suffix="") + message = str(excinfo.value) + assert "end_of_word_suffix" in message + assert "not implemented" in message.lower() + + +# -- cache_capacity is accepted, not rejected ------------------------------ + + +def test_cache_capacity_is_retained(): + # A cache is a pure optimization: its absence cannot change output, so this + # is accepted rather than rejected. It was previously discarded entirely. + assert BPE(cache_capacity=1234).cache_capacity == 1234 + + +def test_cache_capacity_stays_out_of_serialization(): + # HuggingFace does not serialize cache_capacity into tokenizer.json, so + # omitting it keeps to_dict() faithful rather than lossy. + assert "cache_capacity" not in BPE(cache_capacity=1234).to_dict() + + +# -- deserialization path -------------------------------------------------- + + +def test_unsupported_option_in_tokenizer_json_is_rejected_on_load(): + from ubi_tokenizer.tokenizer import Tokenizer + + payload = { + "version": "1.0", + "added_tokens": [], + "model": {"type": "BPE", "vocab": {"a": 0}, "merges": [], "dropout": 0.5}, + } + with pytest.raises(UnsupportedFeatureError, match="dropout"): + Tokenizer.from_dict(payload)