feat(tokenizer): implement deterministic dataset tokenization with Merkle verification

Legacy/openverifiablellm/tokenizer/__init__.py

@@ -1,6 +1,18 @@
+from .base import BaseTokenizer
+from .bpe_tokenizer import BPETokenizer
+from .factory import create_tokenizer, load_tokenizer
+from .sentencepiece_tokenizer import SentencePieceTokenizer
+from .tokenize_dataset import tokenize_dataset, verify_tokenized_dataset
 from .train import hash_tokenizer_config, train_tokenizer
 
 __all__ = [
+    "BaseTokenizer",
+    "BPETokenizer",
+    "SentencePieceTokenizer",
+    "create_tokenizer",
+    "load_tokenizer",
+    "tokenize_dataset",
+    "verify_tokenized_dataset",
     "train_tokenizer",
     "hash_tokenizer_config",
 ]

Legacy/openverifiablellm/tokenizer/base.py

@@ -1,5 +1,6 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
+from typing import List, Optional, Union
 
 
 class BaseTokenizer(ABC):
@@ -22,10 +23,25 @@ def train(self, text_file: Path, save_path: Path):
         """Train tokenizer and save model."""
         pass
 
+    @abstractmethod
+    def load(self, tokenizer_dir: Path):
+        """Load trained tokenizer model from directory."""
+        pass
+
+    @abstractmethod
+    def encode(self, text: str) -> List[int]:
+        """Encode text into a list of token IDs."""
+        pass
+
+    @abstractmethod
+    def decode(self, token_ids: List[int]) -> str:
+        """Decode a list of token IDs back into text."""
+        pass
+
     @abstractmethod
     def get_vocab_path(self, tokenizer_dir: Path) -> Path:
         pass
 
     @abstractmethod
-    def get_merges_path(self, tokenizer_dir: Path):
+    def get_merges_path(self, tokenizer_dir: Path) -> Optional[Path]:
         pass

Legacy/openverifiablellm/tokenizer/bpe_tokenizer.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import List, Optional
 
 from tokenizers import ByteLevelBPETokenizer
 
@@ -8,7 +9,17 @@
 
 
 class BPETokenizer(BaseTokenizer):
+    def __init__(self, vocab_size: int = 32000, min_frequency: int = 2):
+        super().__init__(vocab_size, min_frequency)
+        self._tokenizer: Optional[ByteLevelBPETokenizer] = None
+
     def train(self, text_file: Path, save_path: Path):
+        text_file = Path(text_file)
+        save_path = Path(save_path)
+        if not text_file.is_file():
+            raise FileNotFoundError(f"Text file not found: {text_file}")
+
+        save_path.mkdir(parents=True, exist_ok=True)
         tokenizer = ByteLevelBPETokenizer()
 
         tokenizer.train(
@@ -19,9 +30,34 @@ def train(self, text_file: Path, save_path: Path):
         )
 
         tokenizer.save_model(str(save_path))
+        self._tokenizer = tokenizer
+
+    def load(self, tokenizer_dir: Path):
+        tokenizer_dir = Path(tokenizer_dir)
+        vocab_path = self.get_vocab_path(tokenizer_dir)
+        merges_path = self.get_merges_path(tokenizer_dir)
+
+        if not vocab_path.is_file():
+            raise FileNotFoundError(f"vocab.json not found at {vocab_path}")
+        if not merges_path.is_file():
+            raise FileNotFoundError(f"merges.txt not found at {merges_path}")
+
+        self._tokenizer = ByteLevelBPETokenizer.from_file(str(vocab_path), str(merges_path))
+
+    def encode(self, text: str) -> List[int]:
+        if not isinstance(text, str):
+            raise TypeError(f"text must be str, got {type(text).__name__}")
+        if self._tokenizer is None:
+            raise RuntimeError("Tokenizer is not trained or loaded. Call train() or load() first.")
+        return self._tokenizer.encode(text).ids
+
+    def decode(self, token_ids: List[int]) -> str:
+        if self._tokenizer is None:
+            raise RuntimeError("Tokenizer is not trained or loaded. Call train() or load() first.")
+        return self._tokenizer.decode(token_ids)
 
     def get_vocab_path(self, tokenizer_dir: Path) -> Path:
-        return tokenizer_dir / "vocab.json"
+        return Path(tokenizer_dir) / "vocab.json"
 
     def get_merges_path(self, tokenizer_dir: Path) -> Path:
-        return tokenizer_dir / "merges.txt"
+        return Path(tokenizer_dir) / "merges.txt"

Legacy/openverifiablellm/tokenizer/factory.py

@@ -1,8 +1,16 @@
+from pathlib import Path
+from typing import Optional, Union
+
+from .base import BaseTokenizer
 from .bpe_tokenizer import BPETokenizer
 from .sentencepiece_tokenizer import SentencePieceTokenizer
 
 
-def create_tokenizer(tokenizer_type, vocab_size, min_frequency):
+def create_tokenizer(
+    tokenizer_type: str,
+    vocab_size: int = 32000,
+    min_frequency: int = 2,
+) -> BaseTokenizer:
     tokenizer_type = tokenizer_type.lower()
 
     if tokenizer_type == "bpe":
@@ -12,3 +20,41 @@ def create_tokenizer(tokenizer_type, vocab_size, min_frequency):
         return SentencePieceTokenizer(vocab_size, min_frequency)
 
     raise ValueError(f"Unsupported tokenizer: {tokenizer_type}")
+
+
+def load_tokenizer(
+    tokenizer_dir: Union[str, Path],
+    tokenizer_type: Optional[str] = None,
+) -> BaseTokenizer:
+    """
+    Load a trained tokenizer from directory.
+
+    If tokenizer_type is not provided, automatically detects whether
+    SentencePiece (spm.model) or BPE (vocab.json, merges.txt) artifacts are present.
+    """
+    tokenizer_dir = Path(tokenizer_dir)
+    if not tokenizer_dir.is_dir():
+        raise NotADirectoryError(f"Tokenizer directory not found: {tokenizer_dir}")
+
+    t_type = tokenizer_type.lower() if tokenizer_type else None
+
+    has_spm = (tokenizer_dir / "spm.model").is_file()
+    has_bpe = (tokenizer_dir / "vocab.json").is_file()
+
+    if t_type == "sentencepiece" or (t_type is None and has_spm and not has_bpe):
+        tok = SentencePieceTokenizer()
+        tok.load(tokenizer_dir)
+        return tok
+
+    if t_type == "bpe" or (t_type is None and has_bpe):
+        tok = BPETokenizer()
+        tok.load(tokenizer_dir)
+        return tok
+
+    if t_type is not None:
+        raise ValueError(f"Unsupported tokenizer type: {tokenizer_type}")
+
+    raise FileNotFoundError(
+        f"Could not identify tokenizer artifacts in {tokenizer_dir}. "
+        "Expected spm.model for SentencePiece or vocab.json/merges.txt for BPE."
+    )

Legacy/openverifiablellm/tokenizer/sentencepiece_tokenizer.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import List, Optional
 
 import sentencepiece as spm
 
@@ -10,7 +11,17 @@ class SentencePieceTokenizer(BaseTokenizer):
     SentencePiece tokenizer implementation.
     """
 
+    def __init__(self, vocab_size: int = 32000, min_frequency: int = 2):
+        super().__init__(vocab_size, min_frequency)
+        self._sp: Optional[spm.SentencePieceProcessor] = None
+
     def train(self, text_file: Path, save_path: Path):
+        text_file = Path(text_file)
+        save_path = Path(save_path)
+        if not text_file.is_file():
+            raise FileNotFoundError(f"Text file not found: {text_file}")
+
+        save_path.mkdir(parents=True, exist_ok=True)
         model_prefix = save_path / "spm"
 
         spm.SentencePieceTrainer.train(
@@ -19,9 +30,36 @@ def train(self, text_file: Path, save_path: Path):
             vocab_size=self.vocab_size,
         )
 
-    def get_vocab_path(self, tokenizer_dir: Path):
-        return tokenizer_dir / "spm.vocab"
+        model_file = save_path / "spm.model"
+        if model_file.is_file():
+            self._sp = spm.SentencePieceProcessor(model_file=str(model_file))
+
+    def load(self, tokenizer_dir: Path):
+        tokenizer_dir = Path(tokenizer_dir)
+        model_file = self.get_model_path(tokenizer_dir)
+        if not model_file.is_file():
+            raise FileNotFoundError(f"spm.model not found at {model_file}")
+
+        self._sp = spm.SentencePieceProcessor(model_file=str(model_file))
+
+    def encode(self, text: str) -> List[int]:
+        if not isinstance(text, str):
+            raise TypeError(f"text must be str, got {type(text).__name__}")
+        if self._sp is None:
+            raise RuntimeError("Tokenizer is not trained or loaded. Call train() or load() first.")
+        return [int(tok) for tok in self._sp.encode(text, out_type=int)]
+
+    def decode(self, token_ids: List[int]) -> str:
+        if self._sp is None:
+            raise RuntimeError("Tokenizer is not trained or loaded. Call train() or load() first.")
+        return self._sp.decode([int(t) for t in token_ids])
+
+    def get_model_path(self, tokenizer_dir: Path) -> Path:
+        return Path(tokenizer_dir) / "spm.model"
+
+    def get_vocab_path(self, tokenizer_dir: Path) -> Path:
+        return Path(tokenizer_dir) / "spm.vocab"
 
-    def get_merges_path(self, tokenizer_dir: Path):
+    def get_merges_path(self, tokenizer_dir: Path) -> Optional[Path]:
         # SentencePiece does not use merges
         return None