fix(tokenizer): support SentencePiece artifacts in hash_tokenizer_config and add unit tests

Legacy/openverifiablellm/tokenizer/train.py

@@ -1,7 +1,7 @@
 import json
 import logging
 from pathlib import Path
-from typing import Union
+from typing import Optional, Union
 
 from openverifiablellm.utils import compute_sha256
 
@@ -63,32 +63,76 @@ def train_tokenizer(
     return save_path
 
 
-def hash_tokenizer_config(tokenizer_path: Union[str, Path]) -> dict:
+def hash_tokenizer_config(
+    tokenizer_path: Union[str, Path],
+    tokenizer_type: Optional[str] = None,
+) -> dict:
     """
     Compute SHA256 hashes of tokenizer configuration files.
-    """
 
+    Supports both BPE (vocab.json, merges.txt) and SentencePiece (spm.model, spm.vocab).
+    Automatically detects tokenizer type from artifacts present on disk if not explicitly specified.
+    """
     tokenizer_path = Path(tokenizer_path)
 
-    vocab_path = tokenizer_path / "vocab.json"
-    merges_path = tokenizer_path / "merges.txt"
+    t_type = tokenizer_type.lower() if tokenizer_type else None
+
+    has_spm = (tokenizer_path / "spm.model").is_file() or (tokenizer_path / "spm.vocab").is_file()
+    has_bpe = (tokenizer_path / "vocab.json").is_file() or (tokenizer_path / "merges.txt").is_file()
+
+    if t_type == "sentencepiece" or (t_type is None and has_spm and not has_bpe):
+        model_path = tokenizer_path / "spm.model"
+        vocab_path = tokenizer_path / "spm.vocab"
+
+        if not model_path.is_file():
+            raise FileNotFoundError(f"spm.model not found at {model_path}")
+        if not vocab_path.is_file():
+            raise FileNotFoundError(f"spm.vocab not found at {vocab_path}")
+
+        vocab_bytes = vocab_path.read_bytes()
+        vocab_hash = compute_sha256(data=vocab_bytes)
+        vocab_lines = [
+            line
+            for line in vocab_bytes.decode("utf-8", errors="replace").splitlines()
+            if line.strip()
+        ]
+        actual_vocab_size = len(vocab_lines)
+
+        model_hash = compute_sha256(file_path=model_path)
+
+        logger.info("SentencePiece tokenizer config hashed successfully")
+
+        return {
+            "tokenizer_type": "sentencepiece",
+            "tokenizer_vocab_hash": vocab_hash,
+            "tokenizer_model_hash": model_hash,
+            "tokenizer_merges_hash": None,
+            "tokenizer_vocab_size": actual_vocab_size,
+        }
+
+    elif t_type == "bpe" or (t_type is None and (has_bpe or not has_spm)):
+        vocab_path = tokenizer_path / "vocab.json"
+        merges_path = tokenizer_path / "merges.txt"
 
-    if not vocab_path.is_file():
-        raise FileNotFoundError(f"vocab.json not found at {vocab_path}")
+        if not vocab_path.is_file():
+            raise FileNotFoundError(f"vocab.json not found at {vocab_path}")
+        if not merges_path.is_file():
+            raise FileNotFoundError(f"merges.txt not found at {merges_path}")
 
-    if not merges_path.is_file():
-        raise FileNotFoundError(f"merges.txt not found at {merges_path}")
+        vocab_bytes = vocab_path.read_bytes()
+        vocab_hash = compute_sha256(data=vocab_bytes)
+        actual_vocab_size = len(json.loads(vocab_bytes.decode("utf-8")))
 
-    vocab_bytes = vocab_path.read_bytes()
-    vocab_hash = compute_sha256(data=vocab_bytes)
-    actual_vocab_size = len(json.loads(vocab_bytes.decode("utf-8")))
+        merges_hash = compute_sha256(file_path=merges_path)
 
-    merges_hash = compute_sha256(file_path=merges_path)
+        logger.info("BPE tokenizer config hashed successfully")
 
-    logger.info("Tokenizer config hashed successfully")
+        return {
+            "tokenizer_type": "bpe",
+            "tokenizer_vocab_hash": vocab_hash,
+            "tokenizer_merges_hash": merges_hash,
+            "tokenizer_vocab_size": actual_vocab_size,
+        }
 
-    return {
-        "tokenizer_vocab_hash": vocab_hash,
-        "tokenizer_merges_hash": merges_hash,
-        "tokenizer_vocab_size": actual_vocab_size,
-    }
+    else:
+        raise ValueError(f"Unsupported or ambiguous tokenizer type: {tokenizer_type}")

Legacy/tests/test_tokenizer.py

@@ -1,11 +1,15 @@
 import json
 
 import pytest
+import sentencepiece as spm
 
 from openverifiablellm.tokenizer import (
     hash_tokenizer_config,
     train_tokenizer,
 )
+from openverifiablellm.tokenizer.bpe_tokenizer import BPETokenizer
+from openverifiablellm.tokenizer.factory import create_tokenizer
+from openverifiablellm.tokenizer.sentencepiece_tokenizer import SentencePieceTokenizer
 
 
 @pytest.fixture
@@ -41,6 +45,22 @@ def trained_tokenizer(tmp_path, sample_text_file):
     return tokenizer_path
 
 
+@pytest.fixture
+def trained_sentencepiece_tokenizer(tmp_path, sample_text_file):
+    """Train a SentencePiece tokenizer on sample text and return the path."""
+    tokenizer_path = tmp_path / "spm_tokenizer"
+
+    train_tokenizer(
+        text_file=sample_text_file,
+        save_path=tokenizer_path,
+        tokenizer_type="sentencepiece",
+        vocab_size=50,
+        min_frequency=2,
+    )
+
+    return tokenizer_path
+
+
 # ---------------------------------------------------------------------
 # Positive Tests
 # ---------------------------------------------------------------------
@@ -166,3 +186,132 @@ def test_hash_tokenizer_missing_merges(tmp_path):
 
     with pytest.raises(FileNotFoundError):
         hash_tokenizer_config(tokenizer_path)
+
+
+# ---------------------------------------------------------------------
+# SentencePiece Tests
+# ---------------------------------------------------------------------
+
+
+def test_train_sentencepiece_tokenizer_creates_files(trained_sentencepiece_tokenizer):
+    """SentencePiece training should create spm.model and spm.vocab."""
+    assert (trained_sentencepiece_tokenizer / "spm.model").is_file()
+    assert (trained_sentencepiece_tokenizer / "spm.vocab").is_file()
+
+
+def test_train_sentencepiece_tokenizer_is_deterministic(tmp_path, sample_text_file):
+    """Training twice with SentencePiece on same text should produce identical files."""
+    path1 = tmp_path / "spm1"
+    path2 = tmp_path / "spm2"
+
+    train_tokenizer(sample_text_file, path1, tokenizer_type="sentencepiece", vocab_size=50)
+    train_tokenizer(sample_text_file, path2, tokenizer_type="sentencepiece", vocab_size=50)
+
+    sp1_proc = spm.SentencePieceProcessor(model_file=str(path1 / "spm.model"))
+    sp2_proc = spm.SentencePieceProcessor(model_file=str(path2 / "spm.model"))
+
+    assert sp1_proc.get_piece_size() == sp2_proc.get_piece_size()
+    pieces1 = [sp1_proc.id_to_piece(i) for i in range(sp1_proc.get_piece_size())]
+    pieces2 = [sp2_proc.id_to_piece(i) for i in range(sp2_proc.get_piece_size())]
+    assert pieces1 == pieces2
+
+    scores1 = [sp1_proc.get_score(i) for i in range(sp1_proc.get_piece_size())]
+    scores2 = [sp2_proc.get_score(i) for i in range(sp2_proc.get_piece_size())]
+    assert scores1 == scores2
+
+    vocab1 = (path1 / "spm.vocab").read_text(encoding="utf-8")
+    vocab2 = (path2 / "spm.vocab").read_text(encoding="utf-8")
+    assert vocab1 == vocab2
+
+
+def test_hash_tokenizer_config_sentencepiece_returns_hashes(trained_sentencepiece_tokenizer):
+    """Hashing SentencePiece tokenizer should return model and vocab hashes."""
+    hashes = hash_tokenizer_config(trained_sentencepiece_tokenizer)
+
+    assert hashes["tokenizer_type"] == "sentencepiece"
+    assert "tokenizer_vocab_hash" in hashes
+    assert "tokenizer_model_hash" in hashes
+    assert hashes["tokenizer_merges_hash"] is None
+    assert hashes["tokenizer_vocab_size"] > 0
+
+
+def test_hash_changes_when_sentencepiece_vocab_changes(trained_sentencepiece_tokenizer):
+    """Modifying spm.vocab should change its hash."""
+    hashes_before = hash_tokenizer_config(trained_sentencepiece_tokenizer)
+
+    vocab_path = trained_sentencepiece_tokenizer / "spm.vocab"
+    content = vocab_path.read_text(encoding="utf-8")
+    vocab_path.write_text(content + "\nextra_token\t0.0\n", encoding="utf-8")
+
+    hashes_after = hash_tokenizer_config(trained_sentencepiece_tokenizer)
+
+    assert hashes_before["tokenizer_vocab_hash"] != hashes_after["tokenizer_vocab_hash"]
+    assert hashes_after["tokenizer_vocab_size"] == hashes_before["tokenizer_vocab_size"] + 1
+
+
+def test_hash_changes_when_sentencepiece_model_changes(trained_sentencepiece_tokenizer):
+    """Modifying spm.model should change model hash."""
+    hashes_before = hash_tokenizer_config(trained_sentencepiece_tokenizer)
+
+    model_path = trained_sentencepiece_tokenizer / "spm.model"
+    data = bytearray(model_path.read_bytes())
+    data[0] ^= 0xFF
+    model_path.write_bytes(bytes(data))
+
+    hashes_after = hash_tokenizer_config(trained_sentencepiece_tokenizer)
+
+    assert hashes_before["tokenizer_model_hash"] != hashes_after["tokenizer_model_hash"]
+
+
+def test_hash_tokenizer_sentencepiece_missing_model(tmp_path):
+    tokenizer_path = tmp_path / "spm_missing_model"
+    tokenizer_path.mkdir()
+    (tokenizer_path / "spm.vocab").write_text("token\t0.0\n", encoding="utf-8")
+
+    with pytest.raises(FileNotFoundError, match="spm.model"):
+        hash_tokenizer_config(tokenizer_path)
+
+
+def test_hash_tokenizer_sentencepiece_missing_vocab(tmp_path):
+    tokenizer_path = tmp_path / "spm_missing_vocab"
+    tokenizer_path.mkdir()
+    (tokenizer_path / "spm.model").write_bytes(b"dummy_model_bytes")
+
+    with pytest.raises(FileNotFoundError, match="spm.vocab"):
+        hash_tokenizer_config(tokenizer_path)
+
+
+def test_hash_tokenizer_unsupported_type_raises(tmp_path):
+    with pytest.raises(ValueError, match="Unsupported"):
+        hash_tokenizer_config(tmp_path, tokenizer_type="unknown_type")
+
+
+# ---------------------------------------------------------------------
+# create_tokenizer Factory Tests
+# ---------------------------------------------------------------------
+
+
+def test_create_tokenizer_bpe():
+    tok = create_tokenizer("bpe", vocab_size=500, min_frequency=2)
+    assert isinstance(tok, BPETokenizer)
+    assert tok.vocab_size == 500
+    assert tok.min_frequency == 2
+
+
+def test_create_tokenizer_sentencepiece():
+    tok = create_tokenizer("sentencepiece", vocab_size=500, min_frequency=2)
+    assert isinstance(tok, SentencePieceTokenizer)
+    assert tok.vocab_size == 500
+    assert tok.min_frequency == 2
+
+
+def test_create_tokenizer_case_insensitive():
+    assert isinstance(create_tokenizer("BPE", vocab_size=100, min_frequency=1), BPETokenizer)
+    assert isinstance(create_tokenizer("SentencePiece", vocab_size=100, min_frequency=1), SentencePieceTokenizer)
+    assert isinstance(create_tokenizer("sEnTeNcEpIeCe", vocab_size=100, min_frequency=1), SentencePieceTokenizer)
+
+
+def test_create_tokenizer_unsupported_raises():
+    with pytest.raises(ValueError, match="Unsupported tokenizer: invalid_type"):
+        create_tokenizer("invalid_type", vocab_size=100, min_frequency=1)
+

Legacy/tests/test_util.py

@@ -318,6 +318,33 @@ def test_export_and_load_merkle_proof(tmp_path):
     )
 
 
+def test_load_merkle_proof_valid_file(tmp_path):
+    proof_data = {
+        "chunk_index": 0,
+        "chunk_size": 1024,
+        "proof": [["00" * 32, False]],
+    }
+    proof_file = tmp_path / "valid_proof.json"
+    proof_file.write_text(json.dumps(proof_data), encoding="utf-8")
+
+    loaded = utils.load_merkle_proof(proof_file)
+    assert loaded == proof_data
+
+
+def test_load_merkle_proof_file_not_found(tmp_path):
+    with pytest.raises(FileNotFoundError):
+        utils.load_merkle_proof(tmp_path / "non_existent_proof.json")
+
+
+def test_load_merkle_proof_invalid_json(tmp_path):
+    invalid_file = tmp_path / "invalid_proof.json"
+    invalid_file.write_text("not a valid json", encoding="utf-8")
+
+    with pytest.raises(json.JSONDecodeError):
+        utils.load_merkle_proof(invalid_file)
+
+
+
 def test_extract_text_from_xml_malformed_xml(tmp_path, monkeypatch):
     import defusedxml.ElementTree as ET