From 3a6a719a2d16623c6ea16d65cab3ec658c5f7083 Mon Sep 17 00:00:00 2001 From: liuhongda Date: Wed, 5 Aug 2026 07:56:43 +0000 Subject: [PATCH] fix(hunyuan-image3): preserve serialized tokenizer backend Load tokenizer.json as a Tokenizers backend when available so Transformers 5.x cannot silently drop the custom ByteLevel decoder. Fail fast if decoder preservation does not succeed, protecting prompt BPE encoding and COT decoding. --- .../hunyuan_image3/hunyuan_image3_runner.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lightx2v/models/runners/hunyuan_image3/hunyuan_image3_runner.py b/lightx2v/models/runners/hunyuan_image3/hunyuan_image3_runner.py index c11112b5f..5e5abb55d 100644 --- a/lightx2v/models/runners/hunyuan_image3/hunyuan_image3_runner.py +++ b/lightx2v/models/runners/hunyuan_image3/hunyuan_image3_runner.py @@ -123,10 +123,25 @@ def _ensure_pipeline_modules(self): except OSError: self.hunyuan_generation_config = GenerationConfig() + tokenizer_kwargs = {} + tokenizer_json_path = Path(model_path) / "tokenizer.json" + if tokenizer_json_path.is_file(): + # Transformers 5.x may rebuild this custom fast tokenizer from + # partial metadata and silently drop its ByteLevel decoder. Load + # the serialized backend explicitly to preserve the checkpoint's + # original BPE encoding and decoding behavior. + from tokenizers import Tokenizer + + tokenizer_kwargs["tokenizer_object"] = Tokenizer.from_file(str(tokenizer_json_path)) + self.hunyuan_tokenizer = modules["HunyuanImage3TokenizerFast"].from_pretrained( model_path, model_version=self.hunyuan_config.model_version, + **tokenizer_kwargs, ) + + if tokenizer_json_path.is_file() and self.hunyuan_tokenizer.backend_tokenizer.decoder is None: + raise RuntimeError(f"Failed to preserve the ByteLevel tokenizer backend from {tokenizer_json_path}. Refusing to continue because prompt encoding and COT decoding would be corrupted.") self.hunyuan_image_processor = modules["HunyuanImage3ImageProcessor"](self.hunyuan_config) # Newer transformers versions alias Siglip2ImageProcessorFast to the # slow processor, whose default output is a Python list. The upstream