From 7525616b7242d44c13b332610e01c1972c580d86 Mon Sep 17 00:00:00 2001 From: Emmanuel Keuleers Date: Sat, 18 Jul 2026 15:48:03 +0200 Subject: [PATCH] Fix token frequency being discarded in BigramChain.load An unconditional `frequency = 1` on the line after the token check overwrote the parsed value, so `token=True` silently behaved like type-based counting: bigram transition weights never reflected corpus frequencies. Remove the stray reassignment so the token flag works as documented. Co-Authored-By: Claude Fable 5 --- wuggy/utilities/bigramchain.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wuggy/utilities/bigramchain.py b/wuggy/utilities/bigramchain.py index 1549c02..a295884 100644 --- a/wuggy/utilities/bigramchain.py +++ b/wuggy/utilities/bigramchain.py @@ -27,7 +27,6 @@ def load(self, datafile, size=100, cutoff=1, token=False): fields = line.strip('\n\t').split(self.language_plugin.separator) reference, input_sequence, frequency = fields frequency = float(frequency) if token == True else 1 - frequency = 1 sequence = (self.language_plugin.transform( input_sequence, frequency)) n = len(sequence.representation)