diff --git a/Legacy/openverifiablellm/tokenizer/train.py b/Legacy/openverifiablellm/tokenizer/train.py index bc8fcc34..31c42729 100644 --- a/Legacy/openverifiablellm/tokenizer/train.py +++ b/Legacy/openverifiablellm/tokenizer/train.py @@ -1,7 +1,7 @@ import json import logging from pathlib import Path -from typing import Union +from typing import Optional, Union from openverifiablellm.utils import compute_sha256 @@ -63,32 +63,76 @@ def train_tokenizer( return save_path -def hash_tokenizer_config(tokenizer_path: Union[str, Path]) -> dict: +def hash_tokenizer_config( + tokenizer_path: Union[str, Path], + tokenizer_type: Optional[str] = None, +) -> dict: """ Compute SHA256 hashes of tokenizer configuration files. - """ + Supports both BPE (vocab.json, merges.txt) and SentencePiece (spm.model, spm.vocab). + Automatically detects tokenizer type from artifacts present on disk if not explicitly specified. + """ tokenizer_path = Path(tokenizer_path) - vocab_path = tokenizer_path / "vocab.json" - merges_path = tokenizer_path / "merges.txt" + t_type = tokenizer_type.lower() if tokenizer_type else None + + has_spm = (tokenizer_path / "spm.model").is_file() or (tokenizer_path / "spm.vocab").is_file() + has_bpe = (tokenizer_path / "vocab.json").is_file() or (tokenizer_path / "merges.txt").is_file() + + if t_type == "sentencepiece" or (t_type is None and has_spm and not has_bpe): + model_path = tokenizer_path / "spm.model" + vocab_path = tokenizer_path / "spm.vocab" + + if not model_path.is_file(): + raise FileNotFoundError(f"spm.model not found at {model_path}") + if not vocab_path.is_file(): + raise FileNotFoundError(f"spm.vocab not found at {vocab_path}") + + vocab_bytes = vocab_path.read_bytes() + vocab_hash = compute_sha256(data=vocab_bytes) + vocab_lines = [ + line + for line in vocab_bytes.decode("utf-8", errors="replace").splitlines() + if line.strip() + ] + actual_vocab_size = len(vocab_lines) + + model_hash = compute_sha256(file_path=model_path) + + logger.info("SentencePiece tokenizer config hashed successfully") + + return { + "tokenizer_type": "sentencepiece", + "tokenizer_vocab_hash": vocab_hash, + "tokenizer_model_hash": model_hash, + "tokenizer_merges_hash": None, + "tokenizer_vocab_size": actual_vocab_size, + } + + elif t_type == "bpe" or (t_type is None and (has_bpe or not has_spm)): + vocab_path = tokenizer_path / "vocab.json" + merges_path = tokenizer_path / "merges.txt" - if not vocab_path.is_file(): - raise FileNotFoundError(f"vocab.json not found at {vocab_path}") + if not vocab_path.is_file(): + raise FileNotFoundError(f"vocab.json not found at {vocab_path}") + if not merges_path.is_file(): + raise FileNotFoundError(f"merges.txt not found at {merges_path}") - if not merges_path.is_file(): - raise FileNotFoundError(f"merges.txt not found at {merges_path}") + vocab_bytes = vocab_path.read_bytes() + vocab_hash = compute_sha256(data=vocab_bytes) + actual_vocab_size = len(json.loads(vocab_bytes.decode("utf-8"))) - vocab_bytes = vocab_path.read_bytes() - vocab_hash = compute_sha256(data=vocab_bytes) - actual_vocab_size = len(json.loads(vocab_bytes.decode("utf-8"))) + merges_hash = compute_sha256(file_path=merges_path) - merges_hash = compute_sha256(file_path=merges_path) + logger.info("BPE tokenizer config hashed successfully") - logger.info("Tokenizer config hashed successfully") + return { + "tokenizer_type": "bpe", + "tokenizer_vocab_hash": vocab_hash, + "tokenizer_merges_hash": merges_hash, + "tokenizer_vocab_size": actual_vocab_size, + } - return { - "tokenizer_vocab_hash": vocab_hash, - "tokenizer_merges_hash": merges_hash, - "tokenizer_vocab_size": actual_vocab_size, - } + else: + raise ValueError(f"Unsupported or ambiguous tokenizer type: {tokenizer_type}") diff --git a/Legacy/tests/test_tokenizer.py b/Legacy/tests/test_tokenizer.py index 1c43f3cd..8357e3d9 100644 --- a/Legacy/tests/test_tokenizer.py +++ b/Legacy/tests/test_tokenizer.py @@ -1,11 +1,15 @@ import json import pytest +import sentencepiece as spm from openverifiablellm.tokenizer import ( hash_tokenizer_config, train_tokenizer, ) +from openverifiablellm.tokenizer.bpe_tokenizer import BPETokenizer +from openverifiablellm.tokenizer.factory import create_tokenizer +from openverifiablellm.tokenizer.sentencepiece_tokenizer import SentencePieceTokenizer @pytest.fixture @@ -41,6 +45,22 @@ def trained_tokenizer(tmp_path, sample_text_file): return tokenizer_path +@pytest.fixture +def trained_sentencepiece_tokenizer(tmp_path, sample_text_file): + """Train a SentencePiece tokenizer on sample text and return the path.""" + tokenizer_path = tmp_path / "spm_tokenizer" + + train_tokenizer( + text_file=sample_text_file, + save_path=tokenizer_path, + tokenizer_type="sentencepiece", + vocab_size=50, + min_frequency=2, + ) + + return tokenizer_path + + # --------------------------------------------------------------------- # Positive Tests # --------------------------------------------------------------------- @@ -166,3 +186,132 @@ def test_hash_tokenizer_missing_merges(tmp_path): with pytest.raises(FileNotFoundError): hash_tokenizer_config(tokenizer_path) + + +# --------------------------------------------------------------------- +# SentencePiece Tests +# --------------------------------------------------------------------- + + +def test_train_sentencepiece_tokenizer_creates_files(trained_sentencepiece_tokenizer): + """SentencePiece training should create spm.model and spm.vocab.""" + assert (trained_sentencepiece_tokenizer / "spm.model").is_file() + assert (trained_sentencepiece_tokenizer / "spm.vocab").is_file() + + +def test_train_sentencepiece_tokenizer_is_deterministic(tmp_path, sample_text_file): + """Training twice with SentencePiece on same text should produce identical files.""" + path1 = tmp_path / "spm1" + path2 = tmp_path / "spm2" + + train_tokenizer(sample_text_file, path1, tokenizer_type="sentencepiece", vocab_size=50) + train_tokenizer(sample_text_file, path2, tokenizer_type="sentencepiece", vocab_size=50) + + sp1_proc = spm.SentencePieceProcessor(model_file=str(path1 / "spm.model")) + sp2_proc = spm.SentencePieceProcessor(model_file=str(path2 / "spm.model")) + + assert sp1_proc.get_piece_size() == sp2_proc.get_piece_size() + pieces1 = [sp1_proc.id_to_piece(i) for i in range(sp1_proc.get_piece_size())] + pieces2 = [sp2_proc.id_to_piece(i) for i in range(sp2_proc.get_piece_size())] + assert pieces1 == pieces2 + + scores1 = [sp1_proc.get_score(i) for i in range(sp1_proc.get_piece_size())] + scores2 = [sp2_proc.get_score(i) for i in range(sp2_proc.get_piece_size())] + assert scores1 == scores2 + + vocab1 = (path1 / "spm.vocab").read_text(encoding="utf-8") + vocab2 = (path2 / "spm.vocab").read_text(encoding="utf-8") + assert vocab1 == vocab2 + + +def test_hash_tokenizer_config_sentencepiece_returns_hashes(trained_sentencepiece_tokenizer): + """Hashing SentencePiece tokenizer should return model and vocab hashes.""" + hashes = hash_tokenizer_config(trained_sentencepiece_tokenizer) + + assert hashes["tokenizer_type"] == "sentencepiece" + assert "tokenizer_vocab_hash" in hashes + assert "tokenizer_model_hash" in hashes + assert hashes["tokenizer_merges_hash"] is None + assert hashes["tokenizer_vocab_size"] > 0 + + +def test_hash_changes_when_sentencepiece_vocab_changes(trained_sentencepiece_tokenizer): + """Modifying spm.vocab should change its hash.""" + hashes_before = hash_tokenizer_config(trained_sentencepiece_tokenizer) + + vocab_path = trained_sentencepiece_tokenizer / "spm.vocab" + content = vocab_path.read_text(encoding="utf-8") + vocab_path.write_text(content + "\nextra_token\t0.0\n", encoding="utf-8") + + hashes_after = hash_tokenizer_config(trained_sentencepiece_tokenizer) + + assert hashes_before["tokenizer_vocab_hash"] != hashes_after["tokenizer_vocab_hash"] + assert hashes_after["tokenizer_vocab_size"] == hashes_before["tokenizer_vocab_size"] + 1 + + +def test_hash_changes_when_sentencepiece_model_changes(trained_sentencepiece_tokenizer): + """Modifying spm.model should change model hash.""" + hashes_before = hash_tokenizer_config(trained_sentencepiece_tokenizer) + + model_path = trained_sentencepiece_tokenizer / "spm.model" + data = bytearray(model_path.read_bytes()) + data[0] ^= 0xFF + model_path.write_bytes(bytes(data)) + + hashes_after = hash_tokenizer_config(trained_sentencepiece_tokenizer) + + assert hashes_before["tokenizer_model_hash"] != hashes_after["tokenizer_model_hash"] + + +def test_hash_tokenizer_sentencepiece_missing_model(tmp_path): + tokenizer_path = tmp_path / "spm_missing_model" + tokenizer_path.mkdir() + (tokenizer_path / "spm.vocab").write_text("token\t0.0\n", encoding="utf-8") + + with pytest.raises(FileNotFoundError, match="spm.model"): + hash_tokenizer_config(tokenizer_path) + + +def test_hash_tokenizer_sentencepiece_missing_vocab(tmp_path): + tokenizer_path = tmp_path / "spm_missing_vocab" + tokenizer_path.mkdir() + (tokenizer_path / "spm.model").write_bytes(b"dummy_model_bytes") + + with pytest.raises(FileNotFoundError, match="spm.vocab"): + hash_tokenizer_config(tokenizer_path) + + +def test_hash_tokenizer_unsupported_type_raises(tmp_path): + with pytest.raises(ValueError, match="Unsupported"): + hash_tokenizer_config(tmp_path, tokenizer_type="unknown_type") + + +# --------------------------------------------------------------------- +# create_tokenizer Factory Tests +# --------------------------------------------------------------------- + + +def test_create_tokenizer_bpe(): + tok = create_tokenizer("bpe", vocab_size=500, min_frequency=2) + assert isinstance(tok, BPETokenizer) + assert tok.vocab_size == 500 + assert tok.min_frequency == 2 + + +def test_create_tokenizer_sentencepiece(): + tok = create_tokenizer("sentencepiece", vocab_size=500, min_frequency=2) + assert isinstance(tok, SentencePieceTokenizer) + assert tok.vocab_size == 500 + assert tok.min_frequency == 2 + + +def test_create_tokenizer_case_insensitive(): + assert isinstance(create_tokenizer("BPE", vocab_size=100, min_frequency=1), BPETokenizer) + assert isinstance(create_tokenizer("SentencePiece", vocab_size=100, min_frequency=1), SentencePieceTokenizer) + assert isinstance(create_tokenizer("sEnTeNcEpIeCe", vocab_size=100, min_frequency=1), SentencePieceTokenizer) + + +def test_create_tokenizer_unsupported_raises(): + with pytest.raises(ValueError, match="Unsupported tokenizer: invalid_type"): + create_tokenizer("invalid_type", vocab_size=100, min_frequency=1) + diff --git a/Legacy/tests/test_util.py b/Legacy/tests/test_util.py index a2b2d027..8a30cde6 100644 --- a/Legacy/tests/test_util.py +++ b/Legacy/tests/test_util.py @@ -318,6 +318,33 @@ def test_export_and_load_merkle_proof(tmp_path): ) +def test_load_merkle_proof_valid_file(tmp_path): + proof_data = { + "chunk_index": 0, + "chunk_size": 1024, + "proof": [["00" * 32, False]], + } + proof_file = tmp_path / "valid_proof.json" + proof_file.write_text(json.dumps(proof_data), encoding="utf-8") + + loaded = utils.load_merkle_proof(proof_file) + assert loaded == proof_data + + +def test_load_merkle_proof_file_not_found(tmp_path): + with pytest.raises(FileNotFoundError): + utils.load_merkle_proof(tmp_path / "non_existent_proof.json") + + +def test_load_merkle_proof_invalid_json(tmp_path): + invalid_file = tmp_path / "invalid_proof.json" + invalid_file.write_text("not a valid json", encoding="utf-8") + + with pytest.raises(json.JSONDecodeError): + utils.load_merkle_proof(invalid_file) + + + def test_extract_text_from_xml_malformed_xml(tmp_path, monkeypatch): import defusedxml.ElementTree as ET