From 7525616b7242d44c13b332610e01c1972c580d86 Mon Sep 17 00:00:00 2001 From: Emmanuel Keuleers Date: Sat, 18 Jul 2026 15:48:03 +0200 Subject: [PATCH 1/6] Fix token frequency being discarded in BigramChain.load An unconditional `frequency = 1` on the line after the token check overwrote the parsed value, so `token=True` silently behaved like type-based counting: bigram transition weights never reflected corpus frequencies. Remove the stray reassignment so the token flag works as documented. Co-Authored-By: Claude Fable 5 --- wuggy/utilities/bigramchain.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wuggy/utilities/bigramchain.py b/wuggy/utilities/bigramchain.py index 1549c02..a295884 100644 --- a/wuggy/utilities/bigramchain.py +++ b/wuggy/utilities/bigramchain.py @@ -27,7 +27,6 @@ def load(self, datafile, size=100, cutoff=1, token=False): fields = line.strip('\n\t').split(self.language_plugin.separator) reference, input_sequence, frequency = fields frequency = float(frequency) if token == True else 1 - frequency = 1 sequence = (self.language_plugin.transform( input_sequence, frequency)) n = len(sequence.representation) From dd7626a6f2dc167100378a6ac840c5c035f3b70e Mon Sep 17 00:00:00 2001 From: Emmanuel Keuleers Date: Sat, 18 Jul 2026 16:14:38 +0200 Subject: [PATCH 2/6] Reengineer core for efficiency; bump to 2.0.0 BigramChain is rewritten as a layered DAG over interned (position, segment) integer nodes with flat NumPy edge arrays. Filters (attribute/frequency/segmentset) are now boolean edge masks over views that share the node/edge store instead of dict-of-dict copies, clean() is a single backward-reachability sweep instead of a recursive copy-until-fixpoint pass, and generate() is an iterative shuffled DFS instead of nested recursive generators. Generator hot paths: sequence cache and word-lexicon buckets are sets (were O(n) list scans), statistics/output modes resolve via getattr (was eval), matches are built with one-level copies (was deepcopy), and lexicons stream from disk instead of readlines(). Parsed chains are cached in a binary .chain.pkl next to the data file, keyed by data file mtime/size, so the text corpus is parsed once per version. Levenshtein statistics (old20, ned1, ld1nn) now use rapidfuzz batched SIMD distance computation; python-Levenshtein is dropped. Behavioral equivalence with the previous implementation is verified by tests/capture_behavior.py over a committed synthetic language: chain edge dumps (type and token weighting), startkeys, exhaustive generation sets (~870k sequences across 9 references), and all statistics are byte-identical between old and new. Benchmarks (synthetic 40k-word lexicon, M-series Mac): candidate loop w/ mandatory stats 4.6x faster ned1+old20 statistics 4.1x faster frequency band filtering 3.5x faster chain load (warm cache) 8.8x faster raw sequence generation 1.4x faster Major version bump: internals and dependencies change (numpy+rapidfuzz replace python-Levenshtein), all frequencies are floats now, word lexicon buckets are sets, and generate_classic match dicts are no longer deep-copied. Co-Authored-By: Claude Fable 5 --- .gitignore | 5 +- dev-requirements.txt | 3 +- requirements.txt | 3 +- setup.py | 2 +- tests/__init__.py | 0 tests/capture_behavior.py | 167 +++++++++++ tests/synthetic_plugin/__init__.py | 0 tests/synthetic_plugin/data.txt | 220 ++++++++++++++ tests/synthetic_plugin/lexicon.txt | 220 ++++++++++++++ tests/synthetic_plugin/lookup.txt | 220 ++++++++++++++ tests/synthetic_plugin/make_data.py | 58 ++++ tests/synthetic_plugin/synthetic.py | 12 + wuggy/evaluators/ld1nn.py | 2 +- wuggy/generators/wuggygenerator.py | 142 +++++---- wuggy/plugins/baselanguageplugin.py | 29 +- wuggy/utilities/bigramchain.py | 450 +++++++++++++++++++++------- 16 files changed, 1359 insertions(+), 174 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/capture_behavior.py create mode 100644 tests/synthetic_plugin/__init__.py create mode 100644 tests/synthetic_plugin/data.txt create mode 100644 tests/synthetic_plugin/lexicon.txt create mode 100644 tests/synthetic_plugin/lookup.txt create mode 100644 tests/synthetic_plugin/make_data.py create mode 100644 tests/synthetic_plugin/synthetic.py diff --git a/.gitignore b/.gitignore index 734ec00..bce9948 100644 --- a/.gitignore +++ b/.gitignore @@ -146,4 +146,7 @@ build_documentation .DS_Store # Claude Code -.claude/ \ No newline at end of file +.claude/ + +# Wuggy binary chain caches (written next to language data files) +*.chain.pkl diff --git a/dev-requirements.txt b/dev-requirements.txt index 9b1cba5..afbc116 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,9 +1,8 @@ autopep8==2.0.4 pycodestyle==2.11.0 pylint==3.0.3 -python-Levenshtein==0.25.0 setuptools==69.1.0 wheel==0.42.0 twine==5.0.0 pdoc3==0.10.0 -statsmodels==0.14.1 \ No newline at end of file +statsmodels==0.14.1 diff --git a/requirements.txt b/requirements.txt index b0fb0b3..c2eae79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -Levenshtein >= 0.12.0 +numpy >= 1.22 +rapidfuzz >= 3.0 statsmodels >= 0.12.1 diff --git a/setup.py b/setup.py index 4819809..5d1bdf3 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="wuggy", - version="1.2.0", + version="2.0.0", author="Emmanuel Keuleers", author_email="E.A.Keuleers@tilburguniversity.edu", description="Wuggy: A multilingual pseudoword generator", diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/capture_behavior.py b/tests/capture_behavior.py new file mode 100644 index 0000000..a3c1529 --- /dev/null +++ b/tests/capture_behavior.py @@ -0,0 +1,167 @@ +""" +Capture the observable behavior of the Wuggy core as order-independent data. + +Run against two versions of the code and diff the JSON outputs to prove +behavioral equivalence: + + python tests/capture_behavior.py --out baseline.json # on old code + python tests/capture_behavior.py --out new.json # on new code + diff <(jq -S . baseline.json) <(jq -S . new.json) + +Everything recorded is independent of dict ordering and random shuffling: +sorted edge dumps, exhaustive generation sets (hashed), and statistics for +a deterministic sample of candidates. +""" +import argparse +import codecs +import hashlib +import json +import sys +from fractions import Fraction +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO)) +sys.path.insert(1, str(REPO / 'tests')) + +from wuggy import WuggyGenerator # noqa: E402 +from wuggy.utilities.bigramchain import BigramChain # noqa: E402 +from synthetic_plugin.synthetic import SyntheticLanguagePlugin # noqa: E402 + +DATA_DIR = REPO / 'tests' / 'synthetic_plugin' +ENUMERATION_GUARD = 500_000 +STAT_SAMPLE = 150 + + +def jsonable(value): + if isinstance(value, Fraction): + return {'fraction': [value.numerator, value.denominator]} + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + # Normalize: the rewrite stores all frequencies as floats where the + # old code mixed ints and floats. + return round(float(value), 10) + if isinstance(value, dict): + return {str(k): jsonable(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [jsonable(v) for v in value] + return value + + +def canonical(sequence): + """Order-independent canonical string for a generated segment sequence.""" + return '|'.join( + f'{segment.sequence_length}:{segment.segment_length}:{segment.letters}' + for segment in sequence) + + +def edge_dump(chain): + edges = [] + for key, nextkeys in chain.items(): + for nextkey, frequency in nextkeys.items(): + edges.append([key.position, list(key.value), + nextkey.position, list(nextkey.value), + float(frequency)]) + edges.sort() + return edges + + +def startkey_dump(chain): + return sorted([key.position, list(key.value)] for key in chain.startkeys) + + +def enumerate_all(chain): + """Fully drain chain.generate(); returns {canonical: sequence}.""" + sequences = {} + for sequence in chain.generate(): + sequences[canonical(sequence)] = sequence + if len(sequences) > ENUMERATION_GUARD: + raise RuntimeError('enumeration exceeded guard; shrink test data') + return sequences + + +def summarize(sequences): + ordered = sorted(sequences) + return { + 'count': len(ordered), + 'sha256': hashlib.sha256('\n'.join(ordered).encode()).hexdigest(), + 'sample': ordered[:400], + } + + +def capture_statistics(generator, sequences): + """Statistics for a deterministic sample of enumerated sequences.""" + plugin = generator.language_plugin + stats = {} + for canon in sorted(sequences)[:STAT_SAMPLE]: + sequence = sequences[canon] + stats[canon] = { + name: jsonable(getattr(plugin, 'statistic_' + name)(generator, sequence)) + for name in generator.supported_statistics} + return stats + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--out', required=True) + args = parser.parse_args() + + generator = WuggyGenerator() + generator.load('synthetic_test', + local_language_plugin=SyntheticLanguagePlugin()) + result = {'chain_edges_type': edge_dump(generator.bigramchain)} + + token_chain = BigramChain(generator.language_plugin) + with codecs.open(DATA_DIR / 'data.txt', 'r', encoding='utf-8') as f: + token_chain.load(f, token=True) + result['chain_edges_token'] = edge_dump(token_chain) + + lookup = sorted(generator.lookup_lexicon.items()) + one_syllable = [w for w, seg in lookup if '-' not in seg][:5] + two_syllable = [w for w, seg in lookup if '-' in seg][:4] + result['reference_words'] = one_syllable + two_syllable + + result['references'] = {} + for word in result['reference_words']: + segments = generator.lookup_reference_segments(word) + generator.set_reference_sequence(segments) + reference = generator.reference_sequence + maxpos = len(reference) - 1 + entry = { + 'segments': segments, + 'ref_frequencies': jsonable(generator.reference_sequence_frequencies), + 'ref_statistics': jsonable(generator.reference_statistics), + } + + plain = generator.bigramchain.clean(maxpos) + plain.set_startkeys(reference) + entry['startkeys_nofilter'] = startkey_dump(plain) + entry['nofilter'] = summarize(enumerate_all(plain)) + + attr_chain = generator.bigramchain.attribute_filter( + reference, 'segment_length') + attr_clean = attr_chain.clean(maxpos) + attr_clean.set_startkeys(reference) + entry['attr_edges'] = edge_dump(attr_clean) + attr_sequences = enumerate_all(attr_clean) + entry['attr'] = summarize(attr_sequences) + entry['attr_stats'] = capture_statistics(generator, attr_sequences) + + freq_chain = attr_chain.frequency_filter(reference, 4, 4) + freq_clean = freq_chain.clean(maxpos) + freq_clean.set_startkeys(reference) + entry['attr_freq4'] = summarize(enumerate_all(freq_clean)) + + result['references'][word] = entry + + with open(args.out, 'w', encoding='utf-8') as f: + json.dump(result, f, indent=1, sort_keys=True) + counts = [result['references'][w]['nofilter']['count'] + for w in result['reference_words']] + print(f'captured {len(counts)} references, ' + f'enumeration sizes: {counts}', file=sys.stderr) + + +if __name__ == '__main__': + main() diff --git a/tests/synthetic_plugin/__init__.py b/tests/synthetic_plugin/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/synthetic_plugin/data.txt b/tests/synthetic_plugin/data.txt new file mode 100644 index 0000000..655f9d2 --- /dev/null +++ b/tests/synthetic_plugin/data.txt @@ -0,0 +1,220 @@ +baak b:aa:k 4 +bast b:a:st 8 +beelwaast b:ee:l-w:aa:st 4 +beepmiel b:ee:p-m:ie:l 15 +bil b:i:l 4 +book b:oo:k 32 +bund b:u:nd 70 +daagein d:aa:-g:ei:n 4 +dam d:a:m 13 +daziet d:a:-z:ie:t 1 +deis d:ei:s 2 +derman d:e:r-m:a:n 20 +dind d:i:nd 2 +doom d:oo:m 4 +dork d:o:rk 1 +dulstund d:u:l-st:u:nd 11 +dunteel d:u:n-t:ee:l 1 +dup d:u:p 277 +dutwoost d:u:t-w:oo:st 9 +faar f:aa:r 93 +fak f:a:k 6 +fakkrark f:a:k-kr:a:rk 12 +fan f:a:n 8 +fapstel f:a:p-st:e:l 3 +fekroop f:e:-kr:oo:p 2 +fi f:i: 2 +filfur f:i:l-f:u:r 45 +foo f:oo: 1 +foond f:oo:nd 39 +foorkpam f:oo:rk-p:a:m 6 +fu f:u: 1 +fust f:u:st 7 +gaan g:aa:n 2 +geikzeim g:ei:k-z:ei:m 7 +gerkpork g:e:rk-p:o:rk 3 +giengeem g:ie:n-g:ee:m 1 +gikei g:i:-k:ei: 7 +gond g:o:nd 1 +goolzup g:oo:l-z:u:p 9 +goonstiep g:oo:n-st:ie:p 27 +goorknand g:oo:rk-n:a:nd 4 +gund g:u:nd 3 +kaam k:aa:m 42 +kaasnool k:aa:s-n:oo:l 6 +kaastpoom k:aa:st-p:oo:m 6 +kark k:a:rk 2 +kees k:ee:s 26 +keest k:ee:st 1 +keipfust k:ei:p-f:u:st 12 +keist k:ei:st 3 +kel k:e:l 4 +kingul k:i:n-g:u:l 28 +kondreet k:o:nd-r:ee:t 2 +koorplaand k:oo:r-pl:aa:nd 24 +kooskri k:oo:s-kr:i: 1 +krannoos kr:a:n-n:oo:s 2 +kreendkrir kr:ee:nd-kr:i:r 175 +kreest kr:ee:st 12 +kreikraat kr:ei:-kr:aa:t 4 +kreilfit kr:ei:l-f:i:t 18 +kreind kr:ei:nd 9 +krorwo kr:o:r-w:o: 6 +krusfien kr:u:s-f:ie:n 4 +leendstir l:ee:nd-st:i:r 7 +leenmu l:ee:n-m:u: 1 +leerkstin l:ee:rk-st:i:n 14 +leitwoo l:ei:t-w:oo: 109 +lendsnek l:e:nd-sn:e:k 101 +lielres l:ie:l-r:e:s 8 +lirlam l:i:r-l:a:m 11 +listdaal l:i:st-d:aa:l 16 +listkroo l:i:st-kr:oo: 2 +lutweil l:u:t-w:ei:l 1 +maaklein m:aa:k-l:ei:n 46 +maan m:aa:n 1 +maas m:aa:s 1 +meelweist m:ee:l-w:ei:st 25 +meitvep m:ei:t-v:e:p 4 +miem m:ie:m 9 +miest m:ie:st 12 +mind m:i:nd 15 +mitkroork m:i:t-kr:oo:rk 6 +morkron m:o:r-kr:o:n 8 +motbap m:o:t-b:a:p 21 +naatzaark n:aa:t-z:aa:rk 2 +nabaark n:a:-b:aa:rk 16 +neen n:ee:n 47 +neinen n:ei:-n:e:n 4 +niest n:ie:st 27 +nilplees n:i:l-pl:ee:s 1 +nutsiet n:u:t-s:ie:t 5 +panplund p:a:n-pl:u:nd 13 +park p:a:rk 66 +peend p:ee:nd 4 +peer p:ee:r 14 +petkrie p:e:t-kr:ie: 6 +piek p:ie:k 15 +piep p:ie:p 17 +pierk p:ie:rk 281 +pierkreep p:ie:rk-r:ee:p 166 +piespeem p:ie:s-p:ee:m 4 +plak pl:a:k 18 +plam pl:a:m 4 +plas pl:a:s 127 +plast pl:a:st 1 +pleerpoork pl:ee:r-p:oo:rk 23 +pleissnet pl:ei:s-sn:e:t 146 +plook pl:oo:k 9 +plorzat pl:o:r-z:a:t 5 +pond p:o:nd 1 +pooklies p:oo:k-l:ie:s 12 +poon p:oo:n 8 +poorkpot p:oo:rk-p:o:t 209 +popfeest p:o:p-f:ee:st 4 +porkmoot p:o:rk-m:oo:t 1 +pur p:u:r 4 +puttrast p:u:t-tr:a:st 20 +ra r:a: 5 +raam r:aa:m 1 +raanlool r:aa:n-l:oo:l 8 +rap r:a:p 8 +rat r:a:t 2 +reekgust r:ee:k-g:u:st 17 +reep r:ee:p 28 +reinpip r:ei:n-p:i:p 3 +reipkrust r:ei:p-kr:u:st 2 +reksnup r:e:k-sn:u:p 14 +rorlom r:o:r-l:o:m 2 +rund r:u:nd 6 +rurklip r:u:rk-l:i:p 9 +seindkrik s:ei:nd-kr:i:k 9 +seip s:ei:p 9 +seir s:ei:r 19 +senmeit s:e:n-m:ei:t 13 +sie s:ie: 28 +siep s:ie:p 52 +silkeil s:i:l-k:ei:l 1 +sindpleit s:i:nd-pl:ei:t 45 +snaap sn:aa:p 9 +snaatfeel sn:aa:t-f:ee:l 5 +snal sn:a:l 2 +snarzem sn:a:r-z:e:m 4 +sneerk sn:ee:rk 1 +sneik sn:ei:k 2 +sneir sn:ei:r 1 +sneis sn:ei:s 33 +sneissties sn:ei:s-st:ie:s 139 +snendtreest sn:e:nd-tr:ee:st 6 +snolmeis sn:o:l-m:ei:s 11 +snoop sn:oo:p 11 +sootfies s:oo:t-f:ie:s 1 +staand st:aa:nd 4 +staat st:aa:t 17 +steendruk st:ee:nd-r:u:k 5 +stest st:e:st 1 +stet st:e:t 2 +stimsnie st:i:m-sn:ie: 4 +stitman st:i:t-m:a:n 3 +stitsneik st:i:t-sn:ei:k 3 +stoop st:oo:p 4 +sund s:u:nd 9 +sus s:u:s 1 +taampleit t:aa:m-pl:ei:t 33 +tat t:a:t 73 +tee t:ee: 14 +tep t:e:p 59 +tie t:ie: 129 +tiend t:ie:nd 39 +tiermand t:ie:r-m:a:nd 11 +tierzaam t:ie:r-z:aa:m 1 +tinal t:i:-n:a:l 216 +tivark t:i:-v:a:rk 7 +toop t:oo:p 1 +toopkroos t:oo:p-kr:oo:s 77 +traap tr:aa:p 22 +traarplost tr:aa:r-pl:o:st 2 +traastseir tr:aa:st-s:ei:r 11 +traasttook tr:aa:st-t:oo:k 3 +transeim tr:a:n-s:ei:m 21 +trark tr:a:rk 12 +tree tr:ee: 1 +treist tr:ei:st 14 +trer tr:e:r 5 +trerkbeip tr:e:rk-b:ei:p 24 +triem tr:ie:m 10 +trirk tr:i:rk 152 +trootroo tr:oo:-tr:oo: 5 +tropmiend tr:o:p-m:ie:nd 1 +tror tr:o:r 12 +trorke tr:o:r-k:e: 11 +trurvurk tr:u:r-v:u:rk 9 +truttiend tr:u:t-t:ie:nd 2 +tul t:u:l 6 +tundfoond t:u:nd-f:oo:nd 1 +veerk v:ee:rk 46 +velsnak v:e:l-sn:a:k 1 +videis v:i:-d:ei:s 1 +vum v:u:m 1 +vustwurk v:u:st-w:u:rk 4 +vut v:u:t 102 +waa w:aa: 1 +waal w:aa:l 10 +waamkrim w:aa:m-kr:i:m 19 +waamrap w:aa:m-r:a:p 9 +week w:ee:k 30 +wees w:ee:s 1 +wieksir w:ie:k-s:i:r 4 +wogeer w:o:-g:ee:r 32 +wok w:o:k 3 +wunsnis w:u:n-sn:i:s 9 +wurpein w:u:r-p:ei:n 2 +zaarree z:aa:r-r:ee: 20 +zan z:a:n 6 +zeirk z:ei:rk 4 +zesloond z:e:s-l:oo:nd 7 +zetren z:e:t-r:e:n 7 +zielvet z:ie:l-v:e:t 34 +zies z:ie:s 4 +zoodek z:oo:-d:e:k 2 +zund z:u:nd 31 diff --git a/tests/synthetic_plugin/lexicon.txt b/tests/synthetic_plugin/lexicon.txt new file mode 100644 index 0000000..032b1d3 --- /dev/null +++ b/tests/synthetic_plugin/lexicon.txt @@ -0,0 +1,220 @@ +baak 4 +bast 8 +beelwaast 4 +beepmiel 15 +bil 4 +book 32 +bund 70 +daagein 4 +dam 13 +daziet 1 +deis 2 +derman 20 +dind 2 +doom 4 +dork 1 +dulstund 11 +dunteel 1 +dup 277 +dutwoost 9 +faar 93 +fak 6 +fakkrark 12 +fan 8 +fapstel 3 +fekroop 2 +fi 2 +filfur 45 +foo 1 +foond 39 +foorkpam 6 +fu 1 +fust 7 +gaan 2 +geikzeim 7 +gerkpork 3 +giengeem 1 +gikei 7 +gond 1 +goolzup 9 +goonstiep 27 +goorknand 4 +gund 3 +kaam 42 +kaasnool 6 +kaastpoom 6 +kark 2 +kees 26 +keest 1 +keipfust 12 +keist 3 +kel 4 +kingul 28 +kondreet 2 +koorplaand 24 +kooskri 1 +krannoos 2 +kreendkrir 175 +kreest 12 +kreikraat 4 +kreilfit 18 +kreind 9 +krorwo 6 +krusfien 4 +leendstir 7 +leenmu 1 +leerkstin 14 +leitwoo 109 +lendsnek 101 +lielres 8 +lirlam 11 +listdaal 16 +listkroo 2 +lutweil 1 +maaklein 46 +maan 1 +maas 1 +meelweist 25 +meitvep 4 +miem 9 +miest 12 +mind 15 +mitkroork 6 +morkron 8 +motbap 21 +naatzaark 2 +nabaark 16 +neen 47 +neinen 4 +niest 27 +nilplees 1 +nutsiet 5 +panplund 13 +park 66 +peend 4 +peer 14 +petkrie 6 +piek 15 +piep 17 +pierk 281 +pierkreep 166 +piespeem 4 +plak 18 +plam 4 +plas 127 +plast 1 +pleerpoork 23 +pleissnet 146 +plook 9 +plorzat 5 +pond 1 +pooklies 12 +poon 8 +poorkpot 209 +popfeest 4 +porkmoot 1 +pur 4 +puttrast 20 +ra 5 +raam 1 +raanlool 8 +rap 8 +rat 2 +reekgust 17 +reep 28 +reinpip 3 +reipkrust 2 +reksnup 14 +rorlom 2 +rund 6 +rurklip 9 +seindkrik 9 +seip 9 +seir 19 +senmeit 13 +sie 28 +siep 52 +silkeil 1 +sindpleit 45 +snaap 9 +snaatfeel 5 +snal 2 +snarzem 4 +sneerk 1 +sneik 2 +sneir 1 +sneis 33 +sneissties 139 +snendtreest 6 +snolmeis 11 +snoop 11 +sootfies 1 +staand 4 +staat 17 +steendruk 5 +stest 1 +stet 2 +stimsnie 4 +stitman 3 +stitsneik 3 +stoop 4 +sund 9 +sus 1 +taampleit 33 +tat 73 +tee 14 +tep 59 +tie 129 +tiend 39 +tiermand 11 +tierzaam 1 +tinal 216 +tivark 7 +toop 1 +toopkroos 77 +traap 22 +traarplost 2 +traastseir 11 +traasttook 3 +transeim 21 +trark 12 +tree 1 +treist 14 +trer 5 +trerkbeip 24 +triem 10 +trirk 152 +trootroo 5 +tropmiend 1 +tror 12 +trorke 11 +trurvurk 9 +truttiend 2 +tul 6 +tundfoond 1 +veerk 46 +velsnak 1 +videis 1 +vum 1 +vustwurk 4 +vut 102 +waa 1 +waal 10 +waamkrim 19 +waamrap 9 +week 30 +wees 1 +wieksir 4 +wogeer 32 +wok 3 +wunsnis 9 +wurpein 2 +zaarree 20 +zan 6 +zeirk 4 +zesloond 7 +zetren 7 +zielvet 34 +zies 4 +zoodek 2 +zund 31 diff --git a/tests/synthetic_plugin/lookup.txt b/tests/synthetic_plugin/lookup.txt new file mode 100644 index 0000000..d0e1ce6 --- /dev/null +++ b/tests/synthetic_plugin/lookup.txt @@ -0,0 +1,220 @@ +baak b:aa:k +bast b:a:st +beelwaast b:ee:l-w:aa:st +beepmiel b:ee:p-m:ie:l +bil b:i:l +book b:oo:k +bund b:u:nd +daagein d:aa:-g:ei:n +dam d:a:m +daziet d:a:-z:ie:t +deis d:ei:s +derman d:e:r-m:a:n +dind d:i:nd +doom d:oo:m +dork d:o:rk +dulstund d:u:l-st:u:nd +dunteel d:u:n-t:ee:l +dup d:u:p +dutwoost d:u:t-w:oo:st +faar f:aa:r +fak f:a:k +fakkrark f:a:k-kr:a:rk +fan f:a:n +fapstel f:a:p-st:e:l +fekroop f:e:-kr:oo:p +fi f:i: +filfur f:i:l-f:u:r +foo f:oo: +foond f:oo:nd +foorkpam f:oo:rk-p:a:m +fu f:u: +fust f:u:st +gaan g:aa:n +geikzeim g:ei:k-z:ei:m +gerkpork g:e:rk-p:o:rk +giengeem g:ie:n-g:ee:m +gikei g:i:-k:ei: +gond g:o:nd +goolzup g:oo:l-z:u:p +goonstiep g:oo:n-st:ie:p +goorknand g:oo:rk-n:a:nd +gund g:u:nd +kaam k:aa:m +kaasnool k:aa:s-n:oo:l +kaastpoom k:aa:st-p:oo:m +kark k:a:rk +kees k:ee:s +keest k:ee:st +keipfust k:ei:p-f:u:st +keist k:ei:st +kel k:e:l +kingul k:i:n-g:u:l +kondreet k:o:nd-r:ee:t +koorplaand k:oo:r-pl:aa:nd +kooskri k:oo:s-kr:i: +krannoos kr:a:n-n:oo:s +kreendkrir kr:ee:nd-kr:i:r +kreest kr:ee:st +kreikraat kr:ei:-kr:aa:t +kreilfit kr:ei:l-f:i:t +kreind kr:ei:nd +krorwo kr:o:r-w:o: +krusfien kr:u:s-f:ie:n +leendstir l:ee:nd-st:i:r +leenmu l:ee:n-m:u: +leerkstin l:ee:rk-st:i:n +leitwoo l:ei:t-w:oo: +lendsnek l:e:nd-sn:e:k +lielres l:ie:l-r:e:s +lirlam l:i:r-l:a:m +listdaal l:i:st-d:aa:l +listkroo l:i:st-kr:oo: +lutweil l:u:t-w:ei:l +maaklein m:aa:k-l:ei:n +maan m:aa:n +maas m:aa:s +meelweist m:ee:l-w:ei:st +meitvep m:ei:t-v:e:p +miem m:ie:m +miest m:ie:st +mind m:i:nd +mitkroork m:i:t-kr:oo:rk +morkron m:o:r-kr:o:n +motbap m:o:t-b:a:p +naatzaark n:aa:t-z:aa:rk +nabaark n:a:-b:aa:rk +neen n:ee:n +neinen n:ei:-n:e:n +niest n:ie:st +nilplees n:i:l-pl:ee:s +nutsiet n:u:t-s:ie:t +panplund p:a:n-pl:u:nd +park p:a:rk +peend p:ee:nd +peer p:ee:r +petkrie p:e:t-kr:ie: +piek p:ie:k +piep p:ie:p +pierk p:ie:rk +pierkreep p:ie:rk-r:ee:p +piespeem p:ie:s-p:ee:m +plak pl:a:k +plam pl:a:m +plas pl:a:s +plast pl:a:st +pleerpoork pl:ee:r-p:oo:rk +pleissnet pl:ei:s-sn:e:t +plook pl:oo:k +plorzat pl:o:r-z:a:t +pond p:o:nd +pooklies p:oo:k-l:ie:s +poon p:oo:n +poorkpot p:oo:rk-p:o:t +popfeest p:o:p-f:ee:st +porkmoot p:o:rk-m:oo:t +pur p:u:r +puttrast p:u:t-tr:a:st +ra r:a: +raam r:aa:m +raanlool r:aa:n-l:oo:l +rap r:a:p +rat r:a:t +reekgust r:ee:k-g:u:st +reep r:ee:p +reinpip r:ei:n-p:i:p +reipkrust r:ei:p-kr:u:st +reksnup r:e:k-sn:u:p +rorlom r:o:r-l:o:m +rund r:u:nd +rurklip r:u:rk-l:i:p +seindkrik s:ei:nd-kr:i:k +seip s:ei:p +seir s:ei:r +senmeit s:e:n-m:ei:t +sie s:ie: +siep s:ie:p +silkeil s:i:l-k:ei:l +sindpleit s:i:nd-pl:ei:t +snaap sn:aa:p +snaatfeel sn:aa:t-f:ee:l +snal sn:a:l +snarzem sn:a:r-z:e:m +sneerk sn:ee:rk +sneik sn:ei:k +sneir sn:ei:r +sneis sn:ei:s +sneissties sn:ei:s-st:ie:s +snendtreest sn:e:nd-tr:ee:st +snolmeis sn:o:l-m:ei:s +snoop sn:oo:p +sootfies s:oo:t-f:ie:s +staand st:aa:nd +staat st:aa:t +steendruk st:ee:nd-r:u:k +stest st:e:st +stet st:e:t +stimsnie st:i:m-sn:ie: +stitman st:i:t-m:a:n +stitsneik st:i:t-sn:ei:k +stoop st:oo:p +sund s:u:nd +sus s:u:s +taampleit t:aa:m-pl:ei:t +tat t:a:t +tee t:ee: +tep t:e:p +tie t:ie: +tiend t:ie:nd +tiermand t:ie:r-m:a:nd +tierzaam t:ie:r-z:aa:m +tinal t:i:-n:a:l +tivark t:i:-v:a:rk +toop t:oo:p +toopkroos t:oo:p-kr:oo:s +traap tr:aa:p +traarplost tr:aa:r-pl:o:st +traastseir tr:aa:st-s:ei:r +traasttook tr:aa:st-t:oo:k +transeim tr:a:n-s:ei:m +trark tr:a:rk +tree tr:ee: +treist tr:ei:st +trer tr:e:r +trerkbeip tr:e:rk-b:ei:p +triem tr:ie:m +trirk tr:i:rk +trootroo tr:oo:-tr:oo: +tropmiend tr:o:p-m:ie:nd +tror tr:o:r +trorke tr:o:r-k:e: +trurvurk tr:u:r-v:u:rk +truttiend tr:u:t-t:ie:nd +tul t:u:l +tundfoond t:u:nd-f:oo:nd +veerk v:ee:rk +velsnak v:e:l-sn:a:k +videis v:i:-d:ei:s +vum v:u:m +vustwurk v:u:st-w:u:rk +vut v:u:t +waa w:aa: +waal w:aa:l +waamkrim w:aa:m-kr:i:m +waamrap w:aa:m-r:a:p +week w:ee:k +wees w:ee:s +wieksir w:ie:k-s:i:r +wogeer w:o:-g:ee:r +wok w:o:k +wunsnis w:u:n-sn:i:s +wurpein w:u:r-p:ei:n +zaarree z:aa:r-r:ee: +zan z:a:n +zeirk z:ei:rk +zesloond z:e:s-l:oo:nd +zetren z:e:t-r:e:n +zielvet z:ie:l-v:e:t +zies z:ie:s +zoodek z:oo:-d:e:k +zund z:u:nd diff --git a/tests/synthetic_plugin/make_data.py b/tests/synthetic_plugin/make_data.py new file mode 100644 index 0000000..c828099 --- /dev/null +++ b/tests/synthetic_plugin/make_data.py @@ -0,0 +1,58 @@ +""" +Deterministically generate a synthetic language for tests and benchmarks. + +Words are built from CV(C) syllables segmented as onset:nucleus:coda, the +same shape official orthographic plugins use with copy_onc. Running this +script regenerates the committed data files bit-for-bit (fixed seed). +""" +import argparse +import random +from pathlib import Path + +ONSETS = ['b', 'k', 'd', 'f', 'g', 'l', 'm', 'n', 'p', 'r', + 's', 't', 'v', 'w', 'z', 'st', 'tr', 'pl', 'kr', 'sn'] +NUCLEI = ['a', 'e', 'i', 'o', 'u', 'aa', 'ee', 'oo', 'ie', 'ei'] +CODAS = ['', 'k', 'l', 'm', 'n', 'p', 'r', 's', 't', 'st', 'rk', 'nd'] + + +def make_words(n, seed=1234, max_syllables=2): + rng = random.Random(seed) + words = {} + while len(words) < n: + nsyllables = rng.randint(1, max_syllables) + syllables = [(rng.choice(ONSETS), rng.choice(NUCLEI), rng.choice(CODAS)) + for _ in range(nsyllables)] + plain = ''.join(''.join(syllable) for syllable in syllables) + if plain in words: + continue + segmented = '-'.join(':'.join(syllable) for syllable in syllables) + frequency = max(1, int(rng.lognormvariate(2.0, 1.5))) + words[plain] = (segmented, frequency) + return words + + +def write_language(words, outdir): + outdir = Path(outdir) + outdir.mkdir(parents=True, exist_ok=True) + items = sorted(words.items()) + with open(outdir / 'data.txt', 'w', encoding='utf-8') as f: + for plain, (segmented, frequency) in items: + f.write(f'{plain}\t{segmented}\t{frequency}\n') + with open(outdir / 'lookup.txt', 'w', encoding='utf-8') as f: + for plain, (segmented, _) in items: + f.write(f'{plain}\t{segmented}\n') + with open(outdir / 'lexicon.txt', 'w', encoding='utf-8') as f: + for plain, (_, frequency) in items: + f.write(f'{plain}\t{frequency}\n') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--words', type=int, default=220) + parser.add_argument('--seed', type=int, default=1234) + parser.add_argument('--max-syllables', type=int, default=2) + parser.add_argument('--out', default=str(Path(__file__).parent)) + args = parser.parse_args() + write_language( + make_words(args.words, args.seed, args.max_syllables), args.out) + print(f'wrote {args.words} words to {args.out}') diff --git a/tests/synthetic_plugin/synthetic.py b/tests/synthetic_plugin/synthetic.py new file mode 100644 index 0000000..f58fc24 --- /dev/null +++ b/tests/synthetic_plugin/synthetic.py @@ -0,0 +1,12 @@ +from wuggy.plugins.baselanguageplugin import BaseLanguagePlugin + + +class SyntheticLanguagePlugin(BaseLanguagePlugin): + """Minimal plugin over the generated synthetic language data.""" + default_data = 'data.txt' + default_word_lexicon = 'lexicon.txt' + default_neighbor_lexicon = 'lexicon.txt' + default_lookup_lexicon = 'lookup.txt' + + def transform(self, input_sequence, frequency=1): + return self.copy_onc(input_sequence, frequency) diff --git a/wuggy/evaluators/ld1nn.py b/wuggy/evaluators/ld1nn.py index 6974d48..bec3978 100644 --- a/wuggy/evaluators/ld1nn.py +++ b/wuggy/evaluators/ld1nn.py @@ -1,6 +1,6 @@ from math import exp -from Levenshtein import distance +from rapidfuzz.distance.Levenshtein import distance def ld1nn(word_sample: [str], diff --git a/wuggy/generators/wuggygenerator.py b/wuggy/generators/wuggygenerator.py index ab4ddd7..9182582 100644 --- a/wuggy/generators/wuggygenerator.py +++ b/wuggy/generators/wuggygenerator.py @@ -1,9 +1,9 @@ import codecs -import copy import importlib import importlib.util import inspect import os +import pickle import sys from collections import defaultdict, namedtuple from csv import writer @@ -55,6 +55,17 @@ def _language_plugins_base_dir() -> Path: from ..utilities.bigramchain import BigramChain +def _copy_statistics(statistics: Dict) -> Dict: + """ + Snapshot a statistics dict for inclusion in a returned match. Values + are scalars or per-call-created containers, so a one-level copy is + enough to decouple the match from the generator's mutable state. + """ + return {key: (dict(value) if isinstance(value, dict) + else list(value) if isinstance(value, list) else value) + for key, value in statistics.items()} + + def _loaded_language_plugin_required(func): """ Decorator used for regular Wuggy methods to ensure that a valid language plugin is loaded before execution. @@ -116,11 +127,11 @@ def __init__(self): self.attribute_filters = {} self.default_attributes = [] self.statistics = {} - self.word_lexicon = defaultdict(list) + self.word_lexicon = defaultdict(set) self.neighbor_lexicon = [] self.reference_statistics = {} self.stat_cache = {} - self.sequence_cache = [] + self.sequence_cache = set() self.difference_statistics = {} self.match_statistics = {} self.lookup_lexicon = {} @@ -179,14 +190,41 @@ def load(self, language_plugin_name: str, if language_plugin_name not in self.bigramchains: default_data_path = os.path.join( self.language_plugin_data_path, language_plugin.default_data) - - data_file = codecs.open(default_data_path, 'r', encoding='utf-8') - self.bigramchains[self.language_plugin_name] = BigramChain( - language_plugin) - self.bigramchains[self.language_plugin_name].load( - data_file) + self.bigramchains[self.language_plugin_name] = ( + self.__load_bigramchain(language_plugin, default_data_path)) self.__activate(self.language_plugin_name) + @staticmethod + def __load_bigramchain(language_plugin, data_path: str) -> BigramChain: + """ + Load a bigram chain from its data file, going through a binary + cache next to the data file so the text is parsed only once per + data file version. Cache failures fall back to parsing silently. + """ + cache_path = data_path + '.chain.pkl' + stat = os.stat(data_path) + source_signature = (stat.st_mtime_ns, stat.st_size) + try: + with open(cache_path, 'rb') as f: + cached = pickle.load(f) + if cached.get('source') == source_signature: + return BigramChain.from_cache_state( + language_plugin, cached['state']) + except (OSError, ValueError, KeyError, pickle.PickleError, + EOFError, AttributeError): + pass + bigramchain = BigramChain(language_plugin) + with open(data_path, 'r', encoding='utf-8') as data_file: + bigramchain.load(data_file) + try: + with open(cache_path, 'wb') as f: + pickle.dump({'source': source_signature, + 'state': bigramchain.to_cache_state()}, + f, protocol=pickle.HIGHEST_PROTOCOL) + except OSError: + pass + return bigramchain + @staticmethod def remove_downloaded_language_plugins() -> None: """ @@ -281,18 +319,16 @@ def __load_word_lexicon(self) -> None: This is currently used internally by __activate only, do not call on your own. """ cutoff = 0 - data_file = codecs.open( - "%s/%s" % (self.language_plugin_data_path, self.language_plugin.default_word_lexicon), - 'r', encoding="utf-8") - self.word_lexicon = defaultdict(list) - lines = data_file.readlines() - for line in lines: - fields = line.strip().split('\t') - word = fields[0] - frequency_per_million = fields[-1] - if float(frequency_per_million) >= cutoff: - self.word_lexicon[word[0], len(word)].append(word) - data_file.close() + self.word_lexicon = defaultdict(set) + with open("%s/%s" % (self.language_plugin_data_path, + self.language_plugin.default_word_lexicon), + 'r', encoding="utf-8") as data_file: + for line in data_file: + fields = line.strip().split('\t') + word = fields[0] + frequency_per_million = fields[-1] + if float(frequency_per_million) >= cutoff: + self.word_lexicon[word[0], len(word)].add(word) def __load_neighbor_lexicon(self) -> None: """ @@ -300,21 +336,16 @@ def __load_neighbor_lexicon(self) -> None: This is currently used internally by __activate only, do not call on your own. """ cutoff = 0 - data_file = codecs.open( - "%s/%s" % - (self.language_plugin_data_path, - self.language_plugin.default_neighbor_lexicon), - 'r', - encoding="utf-8") self.neighbor_lexicon = [] - lines = data_file.readlines() - for line in lines: - fields = line.strip().split('\t') - word = fields[0] - frequency_per_million = fields[-1] - if float(frequency_per_million) >= cutoff: - self.neighbor_lexicon.append(word) - data_file.close() + with open("%s/%s" % (self.language_plugin_data_path, + self.language_plugin.default_neighbor_lexicon), + 'r', encoding="utf-8") as data_file: + for line in data_file: + fields = line.strip().split('\t') + word = fields[0] + frequency_per_million = fields[-1] + if float(frequency_per_million) >= cutoff: + self.neighbor_lexicon.append(word) def __load_lookup_lexicon(self, data_file: bool = None) -> None: """ @@ -323,15 +354,15 @@ def __load_lookup_lexicon(self, data_file: bool = None) -> None: """ self.lookup_lexicon = {} if data_file is None: - data_file = codecs.open( + data_file = open( "%s/%s" % (self.language_plugin_data_path, self.language_plugin.default_lookup_lexicon), 'r', encoding="utf-8") - lines = data_file.readlines() - for line in lines: - fields = line.strip().split(self.language_plugin.separator) + separator = self.language_plugin.separator + for line in data_file: + fields = line.strip().split(separator) reference, representation = fields[0:2] self.lookup_lexicon[reference] = representation data_file.close() @@ -370,7 +401,7 @@ def set_reference_sequence(self, sequence: str) -> None: self.reference_sequence) self.__clear_stat_cache() for name in self.__get_statistics(): - function = eval("self.language_plugin.statistic_%s" % (name)) + function = getattr(self.language_plugin, 'statistic_' + name) self.reference_statistics[name] = function( self, self.reference_sequence) @@ -415,7 +446,7 @@ def apply_statistics(self, sequence: str = None) -> None: if sequence is None: sequence = self.current_sequence for name in self.statistics: - function = eval("self.language_plugin.statistic_%s" % (name)) + function = getattr(self.language_plugin, 'statistic_' + name) if (sequence, name) in self.stat_cache: self.statistics[name] = self.stat_cache[(sequence, name)] else: @@ -444,7 +475,7 @@ def __clear_sequence_cache(self) -> None: """ Clears the sequence cache. Only used by Wuggy internally. """ - self.sequence_cache = [] + self.sequence_cache = set() def list_output_modes(self) -> [str]: """ @@ -460,7 +491,7 @@ def set_output_mode(self, name: str) -> None: """ if name not in self.list_output_modes(): raise ValueError(f"Output mode {name} is not supported.") - self.output_mode = eval("self.language_plugin.output_%s" % (name)) + self.output_mode = getattr(self.language_plugin, 'output_' + name) def set_attribute_filter(self, name: str) -> None: """ @@ -618,15 +649,15 @@ def __generate_classic_inner( # (Re)apply all statistics only if match is found: else search becomes unnecessarily slow self.set_all_statistics() self.apply_statistics() - self.sequence_cache.append( + self.sequence_cache.add( self.language_plugin.output_plain(sequence)) match = {"word": input_sequence, "segments": input_sequence_segments, - "pseudoword": self.output_mode(sequence)} - match.update({"statistics": self.statistics, - "difference_statistics": self.difference_statistics}) - - pseudoword_matches.append(copy.deepcopy(match)) + "pseudoword": self.output_mode(sequence), + "statistics": _copy_statistics(self.statistics), + "difference_statistics": _copy_statistics( + self.difference_statistics)} + pseudoword_matches.append(match) if len(pseudoword_matches) >= ncandidates_per_sequence: return pseudoword_matches @@ -713,14 +744,15 @@ def __generate_gui_inner( continue self.set_all_statistics() self.apply_statistics() - self.sequence_cache.append( + self.sequence_cache.add( self.language_plugin.output_plain(sequence)) match = {"word": input_sequence, "segments": input_sequence_segments, - "pseudoword": self.output_mode(sequence)} - match.update({"statistics": self.statistics, - "difference_statistics": self.difference_statistics}) - pseudoword_matches.append(copy.deepcopy(match)) + "pseudoword": self.output_mode(sequence), + "statistics": _copy_statistics(self.statistics), + "difference_statistics": _copy_statistics( + self.difference_statistics)} + pseudoword_matches.append(match) if len(pseudoword_matches) >= ncandidates_per_sequence: return pseudoword_matches @@ -758,7 +790,7 @@ def generate_advanced(self, clear_cache: bool = True) -> Union[Generator[str, No if self.language_plugin.output_plain(sequence) in self.sequence_cache: pass else: - self.sequence_cache.append( + self.sequence_cache.add( self.language_plugin.output_plain(sequence)) self.current_sequence = sequence self.apply_statistics() diff --git a/wuggy/plugins/baselanguageplugin.py b/wuggy/plugins/baselanguageplugin.py index b8df8de..18eb954 100644 --- a/wuggy/plugins/baselanguageplugin.py +++ b/wuggy/plugins/baselanguageplugin.py @@ -2,8 +2,9 @@ from collections import namedtuple from fractions import Fraction -# Pylint may report no-member error due to C extension -import Levenshtein +import numpy as np +from rapidfuzz import process +from rapidfuzz.distance import Levenshtein def compute_difference(gen_stat, ref_stat): @@ -134,9 +135,11 @@ def _distance(self, source, target): return Levenshtein.distance(source, target) def _old(self, source, lexicon, n): - distances = (distance for neighbor, - distance in self._neighbors(source, lexicon, n)) - return sum(distances) / float(n) + distances = process.cdist( + [source], lexicon, scorer=Levenshtein.distance, workers=-1)[0] + if len(distances) > n: + distances = np.partition(distances, n - 1)[:n] + return float(distances.sum(dtype=np.int64)) / float(n) def _neighbors(self, source, lexicon, n): neighbors = [] @@ -146,13 +149,15 @@ def _neighbors(self, source, lexicon, n): return neighbors[0:n] def _neighbors_at_distance(self, source, lexicon, distance): - neighbors = [] - for target in lexicon: - if abs(len(target) - len(source)) > distance: - pass - elif Levenshtein.distance(source, target) == 1: - neighbors.append(target) - return neighbors + length = len(source) + candidates = [target for target in lexicon + if abs(len(target) - length) <= distance] + if not candidates: + return [] + distances = process.cdist( + [source], candidates, scorer=Levenshtein.distance, + score_cutoff=1, workers=-1)[0] + return [target for target, d in zip(candidates, distances) if d == 1] @match @difference diff --git a/wuggy/utilities/bigramchain.py b/wuggy/utilities/bigramchain.py index a295884..f487e41 100644 --- a/wuggy/utilities/bigramchain.py +++ b/wuggy/utilities/bigramchain.py @@ -1,69 +1,266 @@ import random from collections import defaultdict, namedtuple +import numpy as np + Link = namedtuple('Link', ['position', 'value']) +_SENTINEL = object() + +# Bump when the cached on-disk representation changes shape. +CACHE_FORMAT_VERSION = 1 -class BigramChain(defaultdict): + +class BigramChain: """ - A dictionary storing the next possible value, given a list of input sequences. + A layered DAG over (position, segment) nodes storing the next possible + values, given a list of input sequences. + + Nodes are interned to integer ids; edges live in flat NumPy arrays + (source, destination, frequency) sorted by source position. Filters + return lightweight views that share the node/edge store and differ only + in their boolean edge mask, so no filter ever copies the chain. """ - def __init__(self, language_plugin, data=None, encoding='utf-8', size=100, cutoff=1, token=False): - defaultdict.__init__(self, dict) + def __init__(self, language_plugin, data=None, encoding='utf-8', + size=100, cutoff=1, token=False): self.language_plugin = language_plugin try: self.hidden_sequence = self.language_plugin.hidden_sequence except AttributeError: self.hidden_sequence = False - if data != None: - self.load(data, size=size, cutoff=cutoff, token=token) - self.startkeys = [] + self.startkeys = {} self.limit_frequencies = {} + # Shared node/edge store; populated by load() or shared by views. + self.node_segments = [] + self.node_positions = np.empty(0, dtype=np.int32) + self.node_index = {} + self.edge_src = np.empty(0, dtype=np.int32) + self.edge_dst = np.empty(0, dtype=np.int32) + self.edge_freq = np.empty(0, dtype=np.float64) + self.edge_pos = np.empty(0, dtype=np.int32) + self._position_slices = {} + self.edge_mask = np.empty(0, dtype=bool) + self._edge_lookup = None + self._out_edges = None + if data is not None: + self.load(data, size=size, cutoff=cutoff, token=token) + + # ------------------------------------------------------------------ + # Construction def load(self, datafile, size=100, cutoff=1, token=False): - lines = datafile.readlines() - for i, line in enumerate(lines): - fields = line.strip('\n\t').split(self.language_plugin.separator) + node_index = {} + node_segments = [] + node_positions = [] + edge_weights = {} + separator = self.language_plugin.separator + transform = self.language_plugin.transform + + def intern(position, segment): + key = (position, segment) + node_id = node_index.get(key) + if node_id is None: + node_id = len(node_segments) + node_index[key] = node_id + node_segments.append(segment) + node_positions.append(position) + return node_id + + for line in datafile: + fields = line.strip('\n\t').split(separator) reference, input_sequence, frequency = fields frequency = float(frequency) if token == True else 1 - sequence = (self.language_plugin.transform( - input_sequence, frequency)) - n = len(sequence.representation) - if frequency >= cutoff and random.randint(1, 100) <= size: - for j in range(n): - key = Link(j, sequence.representation[j]) - if j+1 < n: - next_key = Link(j+1, sequence.representation[j+1]) - self[key][next_key] = self[key].get( - next_key, 0)+sequence.frequency - else: - pass + sequence = transform(input_sequence, frequency) + representation = sequence.representation + n = len(representation) + if frequency >= cutoff and ( + size >= 100 or random.randint(1, 100) <= size): + previous = intern(0, representation[0]) + for j in range(1, n): + current = intern(j, representation[j]) + edge = (previous, current) + edge_weights[edge] = edge_weights.get( + edge, 0) + sequence.frequency + previous = current datafile.close() + + self.node_index = node_index + self.node_segments = node_segments + self.node_positions = np.array(node_positions, dtype=np.int32) + nedges = len(edge_weights) + src = np.empty(nedges, dtype=np.int32) + dst = np.empty(nedges, dtype=np.int32) + freq = np.empty(nedges, dtype=np.float64) + for i, ((s, d), f) in enumerate(edge_weights.items()): + src[i] = s + dst[i] = d + freq[i] = f + # Sort edges by source position so clean() can sweep positions + # back-to-front over contiguous slices. + pos = self.node_positions[src] if nedges else np.empty(0, np.int32) + order = np.argsort(pos, kind='stable') + self.edge_src = src[order] + self.edge_dst = dst[order] + self.edge_freq = freq[order] + self.edge_pos = pos[order] + self._build_position_slices() + self.edge_mask = np.ones(nedges, dtype=bool) + self._invalidate_caches() self.set_startkeys() + def to_cache_state(self): + """ + Picklable snapshot of the parsed store. Segments are flattened to + plain tuples because plugin namedtuple classes are class attributes + and cannot be pickled by reference. + """ + return { + 'version': CACHE_FORMAT_VERSION, + 'node_tuples': [tuple(segment) for segment in self.node_segments], + 'node_positions': self.node_positions, + 'edge_src': self.edge_src, + 'edge_dst': self.edge_dst, + 'edge_freq': self.edge_freq, + 'edge_pos': self.edge_pos, + } + + @classmethod + def from_cache_state(cls, language_plugin, state): + if state.get('version') != CACHE_FORMAT_VERSION: + raise ValueError('incompatible cache format') + chain = cls(language_plugin) + nfields = len(language_plugin.Segment._fields) + segments = [] + for values in state['node_tuples']: + segment_cls = (language_plugin.Segment + if len(values) == nfields + else language_plugin.SegmentH) + segments.append(segment_cls(*values)) + chain.node_segments = segments + chain.node_positions = state['node_positions'] + chain.node_index = { + (int(position), segment): node_id + for node_id, (position, segment) + in enumerate(zip(chain.node_positions, segments))} + chain.edge_src = state['edge_src'] + chain.edge_dst = state['edge_dst'] + chain.edge_freq = state['edge_freq'] + chain.edge_pos = state['edge_pos'] + chain._build_position_slices() + chain.edge_mask = np.ones(len(chain.edge_src), dtype=bool) + chain._invalidate_caches() + chain.set_startkeys() + return chain + + def _build_position_slices(self): + self._position_slices = {} + if len(self.edge_pos) == 0: + return + positions = np.unique(self.edge_pos) + starts = np.searchsorted(self.edge_pos, positions, side='left') + ends = np.searchsorted(self.edge_pos, positions, side='right') + for position, start, end in zip(positions, starts, ends): + self._position_slices[int(position)] = slice(int(start), int(end)) + + def _view(self, edge_mask): + """A chain sharing this chain's store with a different edge mask.""" + view = BigramChain.__new__(BigramChain) + view.language_plugin = self.language_plugin + view.hidden_sequence = self.hidden_sequence + view.startkeys = {} + view.limit_frequencies = {} + view.node_segments = self.node_segments + view.node_positions = self.node_positions + view.node_index = self.node_index + view.edge_src = self.edge_src + view.edge_dst = self.edge_dst + view.edge_freq = self.edge_freq + view.edge_pos = self.edge_pos + view._position_slices = self._position_slices + view.edge_mask = edge_mask + view._invalidate_caches() + return view + + def _invalidate_caches(self): + self._edge_lookup = None + self._out_edges = None + + # ------------------------------------------------------------------ + # Derived structures (cached per view) + + def _edges(self): + """{(src_id, dst_id): frequency} over active edges.""" + if self._edge_lookup is None: + active = np.flatnonzero(self.edge_mask) + self._edge_lookup = { + (int(self.edge_src[i]), int(self.edge_dst[i])): + float(self.edge_freq[i]) + for i in active} + return self._edge_lookup + + def _adjacency(self): + """{src_id: [dst_id, ...]} over active edges.""" + if self._out_edges is None: + adjacency = defaultdict(list) + for i in np.flatnonzero(self.edge_mask): + adjacency[int(self.edge_src[i])].append(int(self.edge_dst[i])) + self._out_edges = dict(adjacency) + return self._out_edges + + def _node_id(self, position, segment): + return self.node_index.get((position, segment)) + + # ------------------------------------------------------------------ + # Mapping-style compatibility (cold paths: display, dumps, limits) + + def __len__(self): + return len(self._adjacency()) + + def __contains__(self, key): + node_id = self._node_id(key.position, key.value) + return node_id is not None and node_id in self._adjacency() + + def keys(self): + return [Link(int(self.node_positions[node_id]), + self.node_segments[node_id]) + for node_id in self._adjacency()] + + def items(self): + for node_id, children in self._adjacency().items(): + key = Link(int(self.node_positions[node_id]), + self.node_segments[node_id]) + edges = self._edges() + nextkeys = { + Link(int(self.node_positions[child]), + self.node_segments[child]): + edges[(node_id, child)] + for child in children} + yield key, nextkeys + + # ------------------------------------------------------------------ + # Public interface + def set_startkeys(self, reference_sequence=None, fields=None): - if fields == None: + if fields is None: fields = self.language_plugin.default_fields - if reference_sequence == None: - self.startkeys = dict([(key, 0) - for key in self.keys() if key.position == 0]) - else: - self.startkeys = {} - for key in self.keys(): - if key.position == 0: - self.startkeys[key] = 0 + adjacency = self._adjacency() + self.startkeys = { + Link(0, self.node_segments[node_id]): 0 + for node_id in adjacency + if self.node_positions[node_id] == 0} def get_frequencies(self, reference_sequence): frequencies = {} - for position in range(len(reference_sequence)-1): - key = Link(position, reference_sequence[position]) - nextkey = Link(position+1, reference_sequence[position+1]) - try: - frequency = self[key][nextkey] - except KeyError: - frequency = 0 - frequencies[position] = frequency + edges = self._edges() + for position in range(len(reference_sequence) - 1): + source = self._node_id(position, reference_sequence[position]) + target = self._node_id( + position + 1, reference_sequence[position + 1]) + if source is None or target is None: + frequencies[position] = 0 + else: + frequencies[position] = edges.get((source, target), 0) return frequencies def build_limit_frequencies(self, fields): @@ -71,11 +268,11 @@ def build_limit_frequencies(self, fields): for key, nextkeys in self.items(): position, value = key subkey_a = (position, tuple( - [value.__getattribute__(field) for field in fields])) + getattr(value, field) for field in fields)) for nextkey, frequency in nextkeys.items(): position, value = nextkey subkey_b = (position, tuple( - [value.__getattribute__(field) for field in fields])) + getattr(value, field) for field in fields)) subkey = (subkey_a, subkey_b) minfrequency = limits[subkey].get('min', frequency) limits[subkey]['min'] = min(minfrequency, frequency) @@ -84,87 +281,138 @@ def build_limit_frequencies(self, fields): self.limit_frequencies[tuple(fields)] = limits def frequency_filter(self, reference_sequence, lower, upper, kind='dev'): - result = BigramChain(self.language_plugin) - frequencies = self.get_frequencies(reference_sequence) - for key, nextkeys in self.items(): - try: - if kind == 'dev': - minfreq = frequencies[key.position]-lower - maxfreq = frequencies[key.position]+upper - elif kind == 'limit': - minfreq = lower - maxfreq = upper - except: - pass - else: - for nextkey, frequency in nextkeys.items(): - if minfreq <= frequency <= maxfreq: - result[key][nextkey] = frequency - result = result.clean(len(reference_sequence)-1) + # Concentric search widens bands as 2**k without bound; clamp so the + # arbitrary-precision ints survive the conversion to float64. + lower = float(max(min(lower, 1e300), -1e300)) + upper = float(max(min(upper, 1e300), -1e300)) + nref = len(reference_sequence) + max_position = int(self.edge_pos[-1]) if len(self.edge_pos) else -1 + lower_by_position = np.full(max_position + 2, np.inf) + upper_by_position = np.full(max_position + 2, -np.inf) + if kind == 'dev': + frequencies = self.get_frequencies(reference_sequence) + for position, frequency in frequencies.items(): + if position <= max_position: + lower_by_position[position] = frequency - lower + upper_by_position[position] = frequency + upper + elif kind == 'limit': + limit = min(nref - 1, max_position + 1) + lower_by_position[:limit] = lower + upper_by_position[:limit] = upper + if len(self.edge_pos): + edge_lower = lower_by_position[self.edge_pos] + edge_upper = upper_by_position[self.edge_pos] + mask = (self.edge_mask + & (self.edge_freq >= edge_lower) + & (self.edge_freq <= edge_upper)) + else: + mask = self.edge_mask.copy() + result = self._view(mask)._clean_in_place(nref - 1) result.set_startkeys() return result def segmentset_filter(self, reference_sequence, segmentset): segmentset = segmentset.union(set(('^', '$'))) - result = BigramChain(self.language_plugin) - for key, nextkeys in self.items(): - if key.value.letters in segmentset: - for nextkey, frequency in nextkeys.items(): - if nextkey.value.letters in segmentset: - result[key][nextkey] = frequency - result = result.clean(len(reference_sequence)-1) + node_pass = np.fromiter( + (segment.letters in segmentset for segment in self.node_segments), + dtype=bool, count=len(self.node_segments)) + if len(self.edge_mask): + mask = (self.edge_mask + & node_pass[self.edge_src] & node_pass[self.edge_dst]) + else: + mask = self.edge_mask.copy() + result = self._view(mask)._clean_in_place(len(reference_sequence) - 1) result.set_startkeys() return result def attribute_filter(self, reference_sequence, attribute): - result = BigramChain(self.language_plugin) + nref = len(reference_sequence) if type(reference_sequence[0]) == self.language_plugin.Segment: - for key, nextkeys in self.items(): - try: - if key.value.__getattribute__(attribute) == reference_sequence[ - key.position].__getattribute__(attribute): - result[key] = self[key] - except IndexError: - pass + def reference_value(position): + return getattr(reference_sequence[position], attribute) else: - for key, nextkeys in self.items(): - try: - if key.value.__getattribute__(attribute) == reference_sequence[key.position]: - result[key] = self[key] - except IndexError: - pass - return result + def reference_value(position): + return reference_sequence[position] + node_pass = np.zeros(len(self.node_segments), dtype=bool) + for node_id, segment in enumerate(self.node_segments): + position = int(self.node_positions[node_id]) + if position < nref: + node_pass[node_id] = ( + getattr(segment, attribute) == reference_value(position)) + if len(self.edge_mask): + mask = self.edge_mask & node_pass[self.edge_src] + else: + mask = self.edge_mask.copy() + return self._view(mask) def clean(self, maxpos): """ Remove chains that can not be completed. """ - result = BigramChain(self.language_plugin) - for key, nextkeys in self.items(): - for nextkey, frequency in nextkeys.items(): - if nextkey in self or nextkey.position == maxpos: - result[key][nextkey] = frequency - if len(self) == len(result): - return result - else: - return result.clean(maxpos) + return self._view(self.edge_mask.copy())._clean_in_place(maxpos) + + def _clean_in_place(self, maxpos): + """ + Keep only edges on a path reaching position maxpos: one backward + sweep over positions instead of the old copy-until-fixpoint pass. + An edge survives if its destination is at maxpos or its destination + has a surviving outgoing edge. + """ + mask = self.edge_mask + if len(mask) == 0: + return self + mask[self.edge_pos >= maxpos] = False + alive = self.node_positions == maxpos + for position in range(maxpos - 1, -1, -1): + edge_slice = self._position_slices.get(position) + if edge_slice is None: + continue + keep = mask[edge_slice] & alive[self.edge_dst[edge_slice]] + mask[edge_slice] = keep + alive[self.edge_src[edge_slice][keep]] = True + self._invalidate_caches() + return self def generate(self, startkeys=None): if startkeys is None: startkeys = self.startkeys - startkeys = list(startkeys.items()) - random.shuffle(startkeys) - startkeys = dict(startkeys) - if len(self) > 0: - for key in startkeys: - if key not in self: - yield (key.value,) - else: - next_keys = self[key] - for result in self.generate(next_keys): - yield (key.value,)+result - else: + if len(self) == 0: raise Exception('LinkError') + adjacency = self._adjacency() + segments = self.node_segments + + starts = [] + for key in startkeys: + node_id = self._node_id(key.position, key.value) + if node_id is not None and node_id in adjacency: + starts.append((key.value, node_id)) + else: + starts.append((key.value, None)) + random.shuffle(starts) + + for start_value, start_id in starts: + if start_id is None: + yield (start_value,) + continue + path = [start_id] + children = adjacency[start_id][:] + random.shuffle(children) + stack = [iter(children)] + while stack: + node_id = next(stack[-1], _SENTINEL) + if node_id is _SENTINEL: + stack.pop() + path.pop() + continue + children = adjacency.get(node_id) + if children: + path.append(node_id) + children = children[:] + random.shuffle(children) + stack.append(iter(children)) + else: + yield tuple(segments[i] for i in path) + ( + segments[node_id],) def display(self): for key, nextkeys in sorted(self.items(), key=lambda x: x): From 73700c0c7c22702eea55fc1180f9881fd83271ac Mon Sep 17 00:00:00 2001 From: Emmanuel Keuleers Date: Sat, 18 Jul 2026 16:33:48 +0200 Subject: [PATCH 3/6] Add uniform and weighted generation modes plus exact path counting The DAG representation makes candidate sampling tractable, so generation now supports three modes, exposed as generate(mode=...) on BigramChain, generate_advanced(mode=...), and generate_classic/generate_gui (search_mode=...): exhaustive shuffled depth-first traversal (default; unchanged and still byte-identical to the 1.x behavior) uniform every candidate exactly once, in unbiased random order (uniform sampling without replacement); removes the low-branching-subtree bias of DFS order weighted every candidate exactly once, drawn with probability proportional to the product of its transition frequencies, so the most word-like candidates tend to arrive first Both sampling modes draw without replacement: completion counts/weights are computed once per subgraph by a backward DP sweep, and drawn paths are recorded in a prefix trie whose removed mass is subtracted during the walk (per-prefix, since suffix counts are shared across prefixes). Integer counts guarantee exactly-once yields and termination, which keeps the concentric-search loop's drain-then-widen contract intact. BigramChain.count_paths() reports the exact number of candidates a subgraph can produce without generating them. tests/test_sampling.py verifies set-equality of all three modes over 10 filtered subgraphs, exact count_paths, seeded reproducibility, the weighted-order bias, and generator API integration. Co-Authored-By: Claude Fable 5 --- tests/test_sampling.py | 166 +++++++++++++++++++++ wuggy/generators/wuggygenerator.py | 33 +++-- wuggy/utilities/bigramchain.py | 229 +++++++++++++++++++++++++++-- 3 files changed, 405 insertions(+), 23 deletions(-) create mode 100644 tests/test_sampling.py diff --git a/tests/test_sampling.py b/tests/test_sampling.py new file mode 100644 index 0000000..b202288 --- /dev/null +++ b/tests/test_sampling.py @@ -0,0 +1,166 @@ +""" +Tests for count_paths() and the uniform/weighted generation modes. + +Both sampling modes draw without replacement, so they must yield exactly +the same set of sequences as exhaustive traversal, each exactly once. +Runnable directly: python tests/test_sampling.py +""" +import random +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO)) +sys.path.insert(1, str(REPO / 'tests')) + +from wuggy import WuggyGenerator # noqa: E402 +from synthetic_plugin.synthetic import SyntheticLanguagePlugin # noqa: E402 + +PERMUTATION_SIZE_CAP = 30_000 + + +def make_generator(): + generator = WuggyGenerator() + generator.load('synthetic_test', + local_language_plugin=SyntheticLanguagePlugin()) + return generator + + +def canonical(sequence): + return '|'.join( + f'{s.sequence_length}:{s.segment_length}:{s.letters}' + for s in sequence) + + +def prepared_subchains(generator): + """(label, subchain, reference) for a few filter configurations.""" + lookup = sorted(generator.lookup_lexicon.items()) + references = ([w for w, seg in lookup if '-' not in seg][:3] + + [w for w, seg in lookup if '-' in seg][:2]) + for word in references: + generator.set_reference_sequence( + generator.lookup_reference_segments(word)) + reference = generator.reference_sequence + maxpos = len(reference) - 1 + attr = generator.bigramchain.attribute_filter( + reference, 'segment_length') + cleaned = attr.clean(maxpos) + cleaned.set_startkeys(reference) + yield f'{word}/attr', cleaned, reference + banded = attr.frequency_filter(reference, 4, 4) + banded = banded.clean(maxpos) + banded.set_startkeys(reference) + yield f'{word}/attr+freq4', banded, reference + + +def test_permutation_equivalence_and_counts(): + generator = make_generator() + tested = 0 + for label, subchain, _ in prepared_subchains(generator): + expected = subchain.count_paths() + if expected == 0 or expected > PERMUTATION_SIZE_CAP: + continue + exhaustive = [canonical(s) for s in subchain.generate()] + assert len(exhaustive) == expected, ( + f'{label}: count_paths {expected} != exhaustive {len(exhaustive)}') + for mode in ('uniform', 'weighted'): + drawn = [canonical(s) for s in + subchain.generate(mode=mode, rng=random.Random(7))] + assert len(drawn) == expected, ( + f'{label}/{mode}: yielded {len(drawn)}, expected {expected}') + assert set(drawn) == set(exhaustive), ( + f'{label}/{mode}: sequence sets differ from exhaustive') + tested += 1 + assert tested >= 4, f'only {tested} subchains were small enough to test' + print(f'permutation equivalence ok over {tested} subchains') + + +def test_seeded_determinism(): + generator = make_generator() + for label, subchain, _ in prepared_subchains(generator): + if not 0 < subchain.count_paths() <= PERMUTATION_SIZE_CAP: + continue + for mode in ('uniform', 'weighted'): + first = [canonical(s) for s in + subchain.generate(mode=mode, rng=random.Random(123))] + second = [canonical(s) for s in + subchain.generate(mode=mode, rng=random.Random(123))] + assert first == second, f'{label}/{mode}: seeded runs differ' + break + print('seeded determinism ok') + + +def path_weight(subchain, sequence, reference): + frequencies = subchain.get_frequencies(sequence) + weight = 1.0 + for value in frequencies.values(): + weight *= max(value, 1e-12) + return weight + + +def test_weighted_mode_prefers_frequent_paths(): + generator = make_generator() + for label, subchain, reference in prepared_subchains(generator): + if not 1_000 < subchain.count_paths() <= PERMUTATION_SIZE_CAP: + continue + head = 25 + uniform_weights, weighted_weights = [], [] + for seed in range(20): + drawn = subchain.generate(mode='uniform', + rng=random.Random(seed)) + uniform_weights.extend( + path_weight(subchain, s, reference) + for _, s in zip(range(head), drawn)) + drawn = subchain.generate(mode='weighted', + rng=random.Random(seed)) + weighted_weights.extend( + path_weight(subchain, s, reference) + for _, s in zip(range(head), drawn)) + mean_uniform = sum(uniform_weights) / len(uniform_weights) + mean_weighted = sum(weighted_weights) / len(weighted_weights) + assert mean_weighted > mean_uniform, ( + f'{label}: weighted mode did not prefer frequent paths ' + f'({mean_weighted:.1f} <= {mean_uniform:.1f})') + print(f'weighted-bias ok on {label}: mean path weight ' + f'{mean_weighted:.1f} (weighted) vs {mean_uniform:.1f} (uniform)') + return + raise AssertionError('no subchain of suitable size found') + + +def test_generator_api_integration(): + generator = make_generator() + words = sorted(generator.lookup_lexicon)[:3] + for search_mode in ('uniform', 'weighted'): + random.seed(11) + matches = generator.generate_classic( + words, ncandidates_per_sequence=5, max_search_time_per_sequence=3, + search_mode=search_mode) + assert matches, f'no classic matches in {search_mode} mode' + assert all(m['statistics']['lexicality'] == 'N' for m in matches) + # generate_classic leaves its last concentric frequency filter set; + # clear it so the advanced generator starts from the full chain. + generator.clear_frequency_filter() + generator.clear_attribute_filters() + generator.set_reference_sequence( + generator.lookup_reference_segments(words[0])) + generator.set_attribute_filter('segment_length') + generator.set_output_mode('plain') + random.seed(11) + advanced = list(generator.generate_advanced(mode='uniform')) + assert advanced and len(advanced) == len(set(advanced)) + try: + generator.bigramchain.generate(mode='nonsense') + except ValueError: + pass + else: + raise AssertionError('unknown mode did not raise ValueError') + print(f'generator API integration ok ' + f'({len(advanced)} advanced uniform outputs)') + + +if __name__ == '__main__': + test_permutation_equivalence_and_counts() + test_seeded_determinism() + test_weighted_mode_prefers_frequent_paths() + test_generator_api_integration() + print('all sampling tests passed') diff --git a/wuggy/generators/wuggygenerator.py b/wuggy/generators/wuggygenerator.py index 9182582..3ec1b48 100644 --- a/wuggy/generators/wuggygenerator.py +++ b/wuggy/generators/wuggygenerator.py @@ -558,7 +558,8 @@ def generate_classic( ncandidates_per_sequence: int = 10, max_search_time_per_sequence: int = 10, subsyllabic_segment_overlap_ratio: Union[Fraction, None] = Fraction(2, 3), match_subsyllabic_segment_length: bool = True, match_letter_length: bool = True, - output_mode: str = "plain", concentric_search: bool = True) -> [Dict]: + output_mode: str = "plain", concentric_search: bool = True, + search_mode: str = "exhaustive") -> [Dict]: """ This is the classic method to generate pseudowords using Wuggy and can be called immediately after loading a language plugin. The defaults for this method are similar to those set in the legacy version of Wuggy, resulting in sensible pseudowords. @@ -579,7 +580,9 @@ def generate_classic( output_mode: output mode for pseudowords, constricted by the output modes supported by the currently loaded language plugin. - concentric_search: enable/disable concentric search. Wuggy operates best and fastest when concentric search is enabled. First, the algorithm will try to generate candidates that exactly match the transition frequencies of the reference word. Then the maximal allowed deviation in transition frequencies will increase by powers of 2 (i.e., +/-2, +/-4, +/-8, etc.). + concentric_search: enable/disable concentric search. Wuggy operates best and fastest when concentric search is enabled. The maximal allowed deviation in transition frequencies starts at +/-2 and increases by powers of 2 (i.e., +/-2, +/-4, +/-8, etc.), so the earliest candidates are those whose transition frequencies track the reference word most closely. + + search_mode: order in which candidates are drawn from the search space. "exhaustive" (default) is the classic shuffled depth-first traversal. "uniform" draws candidates in unbiased random order (uniform sampling without replacement). "weighted" draws candidates with probability proportional to the product of their transition frequencies, so the most word-like candidates tend to be found first. All modes visit each candidate at most once. .. include:: ../../documentation/wuggygenerator/generate_classic.md """ pseudoword_matches = [] @@ -591,14 +594,15 @@ def generate_classic( max_search_time_per_sequence, subsyllabic_segment_overlap_ratio, match_subsyllabic_segment_length, - match_letter_length, output_mode, concentric_search)) + match_letter_length, output_mode, concentric_search, + search_mode)) return pseudoword_matches def __generate_classic_inner( self, input_sequence: str, ncandidates_per_sequence: int, max_search_time: int, subsyllabic_segment_overlap_ratio: Union[Fraction, None], match_subsyllabic_segment_length: bool, match_letter_length: bool, output_mode: str, - concentric_search: bool = True): + concentric_search: bool = True, search_mode: str = "exhaustive"): """ Inner method for generate_classic(), which outputs a list of pseudoword matches for an input sequence. Should only be used by WuggyGenerator internally. @@ -629,7 +633,7 @@ def __generate_classic_inner( subchain = self.frequency_subchain subchain = subchain.clean(len(self.reference_sequence) - 1) subchain.set_startkeys(self.reference_sequence) - for sequence in subchain.generate(): + for sequence in subchain.generate(mode=search_mode): # Mandatory statistics before finding a suitable match self.clear_statistics() self.set_statistics(["overlap_ratio", "plain_length", "lexicality"]) @@ -668,7 +672,8 @@ def generate_gui( subsyllabic_segment_overlap_ratio: Union[Fraction, None] = Fraction(2, 3), match_subsyllabic_segment_length: bool = True, match_letter_length: bool = True, output_mode: str = "plain", concentric_search: bool = True, - output_type: str = "pseudowords") -> [Dict]: + output_type: str = "pseudowords", + search_mode: str = "exhaustive") -> [Dict]: """ Variant of generate_classic tailored for GUI use. Identical to generate_classic except for the output_type parameter, which controls @@ -689,14 +694,15 @@ def generate_gui( subsyllabic_segment_overlap_ratio, match_subsyllabic_segment_length, match_letter_length, output_mode, concentric_search, - output_type)) + output_type, search_mode)) return pseudoword_matches def __generate_gui_inner( self, input_sequence: str, ncandidates_per_sequence: int, max_search_time: int, subsyllabic_segment_overlap_ratio: Union[Fraction, None], match_subsyllabic_segment_length: bool, match_letter_length: bool, output_mode: str, - concentric_search: bool, output_type: str): + concentric_search: bool, output_type: str, + search_mode: str = "exhaustive"): self.__clear_sequence_cache() self.clear_attribute_filters() self.clear_frequency_filter() @@ -723,7 +729,7 @@ def __generate_gui_inner( subchain = self.frequency_subchain subchain = subchain.clean(len(self.reference_sequence) - 1) subchain.set_startkeys(self.reference_sequence) - for sequence in subchain.generate(): + for sequence in subchain.generate(mode=search_mode): self.clear_statistics() self.set_statistics(["overlap_ratio", "plain_length", "lexicality"]) if (time() - starttime) >= max_search_time: @@ -757,13 +763,16 @@ def __generate_gui_inner( return pseudoword_matches @_loaded_language_plugin_required_generator - def generate_advanced(self, clear_cache: bool = True) -> Union[Generator[str, None, None], - Generator[tuple, None, None]]: + def generate_advanced(self, clear_cache: bool = True, + mode: str = "exhaustive") -> Union[Generator[str, None, None], + Generator[tuple, None, None]]: """ Creates a custom generator which can be iterated to return generated pseudowords. The generator's settings, such as output statistics, should be set by you before calling this method. If attributes such as \"output_mode\" are not set, sensible defaults are used. Note that this method is for advanced users and may result in unexpected results if handled incorrectly. + Parameters: + mode: order in which sequences are drawn from the search space. "exhaustive" (default) is the classic shuffled depth-first traversal. "uniform" yields every sequence exactly once in unbiased random order. "weighted" yields every sequence exactly once, sampled without replacement proportionally to the product of its transition frequencies, so the most word-like sequences tend to arrive first. .. include:: ../../documentation/wuggygenerator/generate_advanced.md """ if clear_cache: @@ -786,7 +795,7 @@ def generate_advanced(self, clear_cache: bool = True) -> Union[Generator[str, No warn( "No reference sequence was set. Ignore this message if this was intentional.") subchain.set_startkeys() - for sequence in subchain.generate(): + for sequence in subchain.generate(mode=mode): if self.language_plugin.output_plain(sequence) in self.sequence_cache: pass else: diff --git a/wuggy/utilities/bigramchain.py b/wuggy/utilities/bigramchain.py index f487e41..7570196 100644 --- a/wuggy/utilities/bigramchain.py +++ b/wuggy/utilities/bigramchain.py @@ -373,14 +373,8 @@ def _clean_in_place(self, maxpos): self._invalidate_caches() return self - def generate(self, startkeys=None): - if startkeys is None: - startkeys = self.startkeys - if len(self) == 0: - raise Exception('LinkError') - adjacency = self._adjacency() - segments = self.node_segments - + def _resolve_startkeys(self, startkeys, adjacency): + """[(segment value, node id or None if not in the chain), ...]""" starts = [] for key in startkeys: node_id = self._node_id(key.position, key.value) @@ -388,7 +382,84 @@ def generate(self, startkeys=None): starts.append((key.value, node_id)) else: starts.append((key.value, None)) - random.shuffle(starts) + return starts + + def _completion_counts(self, weighted=False): + """ + Per-node number of complete paths (or, when weighted, total path + weight as the product of edge frequencies) from the node to any + terminal node, over active edges. A node with no active outgoing + edges is a terminal and counts as one path of weight 1, matching + generate()'s yield condition. + """ + adjacency = self._adjacency() + nodes = set(adjacency) + for children in adjacency.values(): + nodes.update(children) + edges = self._edges() if weighted else None + counts = {} + for node in sorted( + nodes, key=lambda n: int(self.node_positions[n]), + reverse=True): + children = adjacency.get(node) + if not children: + counts[node] = 1.0 if weighted else 1 + elif weighted: + counts[node] = sum( + edges[(node, child)] * counts[child] + for child in children) + else: + counts[node] = sum(counts[child] for child in children) + return counts + + def count_paths(self, startkeys=None): + """ + Exact number of sequences generate() would yield, without + generating them. Startkeys that are not part of the chain yield a + single one-segment sequence and therefore count as one path. + """ + if startkeys is None: + startkeys = self.startkeys + counts = self._completion_counts() + total = 0 + for _, node_id in self._resolve_startkeys(startkeys, + self._adjacency()): + total += 1 if node_id is None else counts[node_id] + return total + + def generate(self, startkeys=None, mode='exhaustive', rng=random): + """ + Yield complete segment sequences from the (filtered) chain. + + Modes: + 'exhaustive' — depth-first traversal with shuffled edge order + (the classic behavior). Complete, but sequence order is + biased toward low-branching regions of the chain. + 'uniform' — every sequence exactly once, in unbiased random + order (uniform sampling without replacement). + 'weighted' — every sequence exactly once, sampled without + replacement with probability proportional to the product + of its transition frequencies, so the most word-like + sequences tend to arrive first. + """ + if mode == 'uniform': + return self._generate_sampled(startkeys, weighted=False, rng=rng) + if mode == 'weighted': + return self._generate_sampled(startkeys, weighted=True, rng=rng) + if mode != 'exhaustive': + raise ValueError(f"Unknown generation mode {mode!r}") + return self._generate_exhaustive(startkeys, rng) + + def _generate_exhaustive(self, startkeys, rng): + if startkeys is None: + startkeys = self.startkeys + if len(self) == 0: + raise Exception('LinkError') + adjacency = self._adjacency() + segments = self.node_segments + + starts = self._resolve_startkeys(startkeys, adjacency) + rng.shuffle(starts) for start_value, start_id in starts: if start_id is None: @@ -396,7 +467,7 @@ def generate(self, startkeys=None): continue path = [start_id] children = adjacency[start_id][:] - random.shuffle(children) + rng.shuffle(children) stack = [iter(children)] while stack: node_id = next(stack[-1], _SENTINEL) @@ -408,12 +479,148 @@ def generate(self, startkeys=None): if children: path.append(node_id) children = children[:] - random.shuffle(children) + rng.shuffle(children) stack.append(iter(children)) else: yield tuple(segments[i] for i in path) + ( segments[node_id],) + def _generate_sampled(self, startkeys, weighted, rng): + """ + Sampling without replacement over complete paths. + + The completion counts/weights below a node are shared by every + prefix that reaches it, so a drawn path must only be removed for + its own prefix. Removed paths are therefore tracked in a prefix + trie: during a draw, the available mass under child c is the + node's total (count or weight) minus what the trie records as + already drawn through this exact prefix. Integer counts guarantee + each path is yielded exactly once and the generator terminates; + float weights only bias the selection order in weighted mode. + Memory grows with the number of sequences drawn (one trie chain + per draw); use exhaustive mode for memory-free full drains. + """ + if startkeys is None: + startkeys = self.startkeys + if len(self) == 0: + raise Exception('LinkError') + adjacency = self._adjacency() + segments = self.node_segments + edges = self._edges() + counts = self._completion_counts() + weights = self._completion_counts(weighted=True) if weighted else None + + starts = self._resolve_startkeys(startkeys, adjacency) + # Trie node: [removed_count, removed_weight, children_by_dag_node]. + trie_root = [0, 0.0, {}] + singletons_drawn = set() + + def available(trie_children, node_id): + removed = trie_children.get(node_id) + return counts[node_id] - (removed[0] if removed else 0) + + while True: + root_children = trie_root[2] + live = [] + for index, (value, node_id) in enumerate(starts): + if node_id is None: + if index not in singletons_drawn: + live.append((index, value, None, 1, 1.0)) + else: + avail = available(root_children, node_id) + if avail > 0: + removed = root_children.get(node_id) + mass = ((weights[node_id] + - (removed[1] if removed else 0.0)) + if weighted else float(avail)) + live.append((index, value, node_id, avail, mass)) + if not live: + return + + if weighted: + total_mass = sum(entry[4] for entry in live) + pick = rng.random() * max(total_mass, 0.0) + chosen = live[-1] + for entry in live: + pick -= entry[4] + if pick <= 0: + chosen = entry + break + else: + pick = rng.randrange(sum(entry[3] for entry in live)) + chosen = live[-1] + for entry in live: + pick -= entry[3] + if pick < 0: + chosen = entry + break + index, value, node_id = chosen[0], chosen[1], chosen[2] + if node_id is None: + singletons_drawn.add(index) + yield (value,) + continue + + path = [node_id] + current = node_id + trie_cursor = trie_root[2].get(node_id) + while True: + children = adjacency.get(current) + if not children: + break + trie_children = trie_cursor[2] if trie_cursor else {} + live_children = [] + for child in children: + avail = available(trie_children, child) + if avail <= 0: + continue + if weighted: + removed = trie_children.get(child) + mass = edges[(current, child)] * ( + weights[child] + - (removed[1] if removed else 0.0)) + live_children.append((child, avail, mass)) + else: + live_children.append((child, avail, float(avail))) + if weighted: + total_mass = sum(mass for _, _, mass in live_children) + pick = rng.random() * max(total_mass, 0.0) + current = live_children[-1][0] + for child, _, mass in live_children: + pick -= mass + if pick <= 0: + current = child + break + else: + pick = rng.randrange( + sum(avail for _, avail, _ in live_children)) + current = live_children[-1][0] + for child, avail, _ in live_children: + pick -= avail + if pick < 0: + current = child + break + path.append(current) + trie_cursor = (trie_cursor[2].get(current) + if trie_cursor else None) + + # Record the drawn path in the trie: below position i the + # removed suffix weight is the product of the remaining edge + # frequencies (1 at the terminal). + suffix_weights = [1.0] * len(path) + for i in range(len(path) - 2, -1, -1): + suffix_weights[i] = (edges[(path[i], path[i + 1])] + * suffix_weights[i + 1]) + cursor = trie_root + for node, suffix_weight in zip(path, suffix_weights): + child = cursor[2].get(node) + if child is None: + child = [0, 0.0, {}] + cursor[2][node] = child + child[0] += 1 + child[1] += suffix_weight + cursor = child + yield tuple(segments[i] for i in path) + def display(self): for key, nextkeys in sorted(self.items(), key=lambda x: x): print('***', key.position, key.value) From 944e9c3482de9ffc370cae8877d3f2ecf35dbfe9 Mon Sep 17 00:00:00 2001 From: Emmanuel Keuleers Date: Sat, 18 Jul 2026 20:20:11 +0200 Subject: [PATCH 4/6] Rename core structures to graph terminology The implementation now literally is the directed graph described in the Wuggy literature, so the names follow suit. Breaking renames, no compatibility aliases (2.0): BigramChain -> SegmentGraph wuggy/utilities/bigramchain.py -> wuggy/utilities/segmentgraph.py Link -> Vertex (it names a vertex id, not an edge) clean() -> prune() startkeys / set_startkeys -> start_vertices / set_start_vertices WuggyGenerator.bigramchain(s) -> segment_graph(s) attribute/frequency_subchain -> attribute/frequency_subgraph 'LinkError' -> 'EmptyGraphError' cache suffix .chain.pkl -> .graph.pkl (format version bumped, old caches are re-parsed silently) Note for language plugin authors: custom statistics that referenced generator.bigramchain must use generator.segment_graph. Sampling tests and the behavioral equivalence capture are byte-identical before and after the rename. Co-Authored-By: Claude Fable 5 --- .gitignore | 2 +- tests/capture_behavior.py | 48 +-- tests/test_sampling.py | 58 ++-- wuggy/generators/wuggygenerator.py | 96 +++--- wuggy/plugins/baselanguageplugin.py | 2 +- .../{bigramchain.py => segmentgraph.py} | 312 +++++++++--------- 6 files changed, 259 insertions(+), 259 deletions(-) rename wuggy/utilities/{bigramchain.py => segmentgraph.py} (69%) diff --git a/.gitignore b/.gitignore index bce9948..f6294e7 100644 --- a/.gitignore +++ b/.gitignore @@ -149,4 +149,4 @@ build_documentation .claude/ # Wuggy binary chain caches (written next to language data files) -*.chain.pkl +*.graph.pkl diff --git a/tests/capture_behavior.py b/tests/capture_behavior.py index a3c1529..9428e45 100644 --- a/tests/capture_behavior.py +++ b/tests/capture_behavior.py @@ -25,7 +25,7 @@ sys.path.insert(1, str(REPO / 'tests')) from wuggy import WuggyGenerator # noqa: E402 -from wuggy.utilities.bigramchain import BigramChain # noqa: E402 +from wuggy.utilities.segmentgraph import SegmentGraph # noqa: E402 from synthetic_plugin.synthetic import SyntheticLanguagePlugin # noqa: E402 DATA_DIR = REPO / 'tests' / 'synthetic_plugin' @@ -56,9 +56,9 @@ def canonical(sequence): for segment in sequence) -def edge_dump(chain): +def edge_dump(graph): edges = [] - for key, nextkeys in chain.items(): + for key, nextkeys in graph.items(): for nextkey, frequency in nextkeys.items(): edges.append([key.position, list(key.value), nextkey.position, list(nextkey.value), @@ -67,14 +67,14 @@ def edge_dump(chain): return edges -def startkey_dump(chain): - return sorted([key.position, list(key.value)] for key in chain.startkeys) +def start_vertex_dump(graph): + return sorted([key.position, list(key.value)] for key in graph.start_vertices) -def enumerate_all(chain): - """Fully drain chain.generate(); returns {canonical: sequence}.""" +def enumerate_all(graph): + """Fully drain graph.generate(); returns {canonical: sequence}.""" sequences = {} - for sequence in chain.generate(): + for sequence in graph.generate(): sequences[canonical(sequence)] = sequence if len(sequences) > ENUMERATION_GUARD: raise RuntimeError('enumeration exceeded guard; shrink test data') @@ -110,12 +110,12 @@ def main(): generator = WuggyGenerator() generator.load('synthetic_test', local_language_plugin=SyntheticLanguagePlugin()) - result = {'chain_edges_type': edge_dump(generator.bigramchain)} + result = {'chain_edges_type': edge_dump(generator.segment_graph)} - token_chain = BigramChain(generator.language_plugin) + token_graph = SegmentGraph(generator.language_plugin) with codecs.open(DATA_DIR / 'data.txt', 'r', encoding='utf-8') as f: - token_chain.load(f, token=True) - result['chain_edges_token'] = edge_dump(token_chain) + token_graph.load(f, token=True) + result['chain_edges_token'] = edge_dump(token_graph) lookup = sorted(generator.lookup_lexicon.items()) one_syllable = [w for w, seg in lookup if '-' not in seg][:5] @@ -134,24 +134,24 @@ def main(): 'ref_statistics': jsonable(generator.reference_statistics), } - plain = generator.bigramchain.clean(maxpos) - plain.set_startkeys(reference) - entry['startkeys_nofilter'] = startkey_dump(plain) + plain = generator.segment_graph.prune(maxpos) + plain.set_start_vertices(reference) + entry['startkeys_nofilter'] = start_vertex_dump(plain) entry['nofilter'] = summarize(enumerate_all(plain)) - attr_chain = generator.bigramchain.attribute_filter( + attr_subgraph = generator.segment_graph.attribute_filter( reference, 'segment_length') - attr_clean = attr_chain.clean(maxpos) - attr_clean.set_startkeys(reference) - entry['attr_edges'] = edge_dump(attr_clean) - attr_sequences = enumerate_all(attr_clean) + attr_pruned = attr_subgraph.prune(maxpos) + attr_pruned.set_start_vertices(reference) + entry['attr_edges'] = edge_dump(attr_pruned) + attr_sequences = enumerate_all(attr_pruned) entry['attr'] = summarize(attr_sequences) entry['attr_stats'] = capture_statistics(generator, attr_sequences) - freq_chain = attr_chain.frequency_filter(reference, 4, 4) - freq_clean = freq_chain.clean(maxpos) - freq_clean.set_startkeys(reference) - entry['attr_freq4'] = summarize(enumerate_all(freq_clean)) + freq_subgraph = attr_subgraph.frequency_filter(reference, 4, 4) + freq_pruned = freq_subgraph.prune(maxpos) + freq_pruned.set_start_vertices(reference) + entry['attr_freq4'] = summarize(enumerate_all(freq_pruned)) result['references'][word] = entry diff --git a/tests/test_sampling.py b/tests/test_sampling.py index b202288..984469f 100644 --- a/tests/test_sampling.py +++ b/tests/test_sampling.py @@ -32,8 +32,8 @@ def canonical(sequence): for s in sequence) -def prepared_subchains(generator): - """(label, subchain, reference) for a few filter configurations.""" +def prepared_subgraphs(generator): + """(label, subgraph, reference) for a few filter configurations.""" lookup = sorted(generator.lookup_lexicon.items()) references = ([w for w, seg in lookup if '-' not in seg][:3] + [w for w, seg in lookup if '-' in seg][:2]) @@ -42,56 +42,56 @@ def prepared_subchains(generator): generator.lookup_reference_segments(word)) reference = generator.reference_sequence maxpos = len(reference) - 1 - attr = generator.bigramchain.attribute_filter( + attr = generator.segment_graph.attribute_filter( reference, 'segment_length') - cleaned = attr.clean(maxpos) - cleaned.set_startkeys(reference) - yield f'{word}/attr', cleaned, reference + pruned = attr.prune(maxpos) + pruned.set_start_vertices(reference) + yield f'{word}/attr', pruned, reference banded = attr.frequency_filter(reference, 4, 4) - banded = banded.clean(maxpos) - banded.set_startkeys(reference) + banded = banded.prune(maxpos) + banded.set_start_vertices(reference) yield f'{word}/attr+freq4', banded, reference def test_permutation_equivalence_and_counts(): generator = make_generator() tested = 0 - for label, subchain, _ in prepared_subchains(generator): - expected = subchain.count_paths() + for label, subgraph, _ in prepared_subgraphs(generator): + expected = subgraph.count_paths() if expected == 0 or expected > PERMUTATION_SIZE_CAP: continue - exhaustive = [canonical(s) for s in subchain.generate()] + exhaustive = [canonical(s) for s in subgraph.generate()] assert len(exhaustive) == expected, ( f'{label}: count_paths {expected} != exhaustive {len(exhaustive)}') for mode in ('uniform', 'weighted'): drawn = [canonical(s) for s in - subchain.generate(mode=mode, rng=random.Random(7))] + subgraph.generate(mode=mode, rng=random.Random(7))] assert len(drawn) == expected, ( f'{label}/{mode}: yielded {len(drawn)}, expected {expected}') assert set(drawn) == set(exhaustive), ( f'{label}/{mode}: sequence sets differ from exhaustive') tested += 1 - assert tested >= 4, f'only {tested} subchains were small enough to test' - print(f'permutation equivalence ok over {tested} subchains') + assert tested >= 4, f'only {tested} subgraphs were small enough to test' + print(f'permutation equivalence ok over {tested} subgraphs') def test_seeded_determinism(): generator = make_generator() - for label, subchain, _ in prepared_subchains(generator): - if not 0 < subchain.count_paths() <= PERMUTATION_SIZE_CAP: + for label, subgraph, _ in prepared_subgraphs(generator): + if not 0 < subgraph.count_paths() <= PERMUTATION_SIZE_CAP: continue for mode in ('uniform', 'weighted'): first = [canonical(s) for s in - subchain.generate(mode=mode, rng=random.Random(123))] + subgraph.generate(mode=mode, rng=random.Random(123))] second = [canonical(s) for s in - subchain.generate(mode=mode, rng=random.Random(123))] + subgraph.generate(mode=mode, rng=random.Random(123))] assert first == second, f'{label}/{mode}: seeded runs differ' break print('seeded determinism ok') -def path_weight(subchain, sequence, reference): - frequencies = subchain.get_frequencies(sequence) +def path_weight(subgraph, sequence, reference): + frequencies = subgraph.get_frequencies(sequence) weight = 1.0 for value in frequencies.values(): weight *= max(value, 1e-12) @@ -100,21 +100,21 @@ def path_weight(subchain, sequence, reference): def test_weighted_mode_prefers_frequent_paths(): generator = make_generator() - for label, subchain, reference in prepared_subchains(generator): - if not 1_000 < subchain.count_paths() <= PERMUTATION_SIZE_CAP: + for label, subgraph, reference in prepared_subgraphs(generator): + if not 1_000 < subgraph.count_paths() <= PERMUTATION_SIZE_CAP: continue head = 25 uniform_weights, weighted_weights = [], [] for seed in range(20): - drawn = subchain.generate(mode='uniform', + drawn = subgraph.generate(mode='uniform', rng=random.Random(seed)) uniform_weights.extend( - path_weight(subchain, s, reference) + path_weight(subgraph, s, reference) for _, s in zip(range(head), drawn)) - drawn = subchain.generate(mode='weighted', + drawn = subgraph.generate(mode='weighted', rng=random.Random(seed)) weighted_weights.extend( - path_weight(subchain, s, reference) + path_weight(subgraph, s, reference) for _, s in zip(range(head), drawn)) mean_uniform = sum(uniform_weights) / len(uniform_weights) mean_weighted = sum(weighted_weights) / len(weighted_weights) @@ -124,7 +124,7 @@ def test_weighted_mode_prefers_frequent_paths(): print(f'weighted-bias ok on {label}: mean path weight ' f'{mean_weighted:.1f} (weighted) vs {mean_uniform:.1f} (uniform)') return - raise AssertionError('no subchain of suitable size found') + raise AssertionError('no subgraph of suitable size found') def test_generator_api_integration(): @@ -138,7 +138,7 @@ def test_generator_api_integration(): assert matches, f'no classic matches in {search_mode} mode' assert all(m['statistics']['lexicality'] == 'N' for m in matches) # generate_classic leaves its last concentric frequency filter set; - # clear it so the advanced generator starts from the full chain. + # clear it so the advanced generator starts from the full graph. generator.clear_frequency_filter() generator.clear_attribute_filters() generator.set_reference_sequence( @@ -149,7 +149,7 @@ def test_generator_api_integration(): advanced = list(generator.generate_advanced(mode='uniform')) assert advanced and len(advanced) == len(set(advanced)) try: - generator.bigramchain.generate(mode='nonsense') + generator.segment_graph.generate(mode='nonsense') except ValueError: pass else: diff --git a/wuggy/generators/wuggygenerator.py b/wuggy/generators/wuggygenerator.py index 3ec1b48..78972a1 100644 --- a/wuggy/generators/wuggygenerator.py +++ b/wuggy/generators/wuggygenerator.py @@ -52,7 +52,7 @@ def _language_plugins_base_dir() -> Path: ) from ..plugins.baselanguageplugin import BaseLanguagePlugin -from ..utilities.bigramchain import BigramChain +from ..utilities.segmentgraph import SegmentGraph def _copy_statistics(statistics: Dict) -> Dict: @@ -96,8 +96,8 @@ def wrapper(*args, **kwargs): class WuggyGenerator(): def __init__(self): - self.bigramchain = None - self.bigramchains = {} + self.segment_graph = None + self.segment_graphs = {} self.supported_official_language_plugin_names = [ "orthographic_basque", "orthographic_dutch", @@ -116,8 +116,8 @@ def __init__(self): "phonetic_french", "phonetic_italian"] self.__official_language_plugin_repository_url = "https://raw.githubusercontent.com/WuggyCode/wuggy_language_plugin_data/master" - self.attribute_subchain = None - self.frequency_subchain = None + self.attribute_subgraph = None + self.frequency_subgraph = None self.reference_sequence = None self.frequency_filter = None self.current_sequence = None @@ -139,7 +139,7 @@ def __init__(self): def load(self, language_plugin_name: str, local_language_plugin: BaseLanguagePlugin = None) -> None: """ - Loads in a language plugin, if available, and stores the corresponding bigramchains. + Loads in a language plugin, if available, and stores the corresponding segment_graphs. Parameters: language_plugin_name: must be the exact string of an official language plugin (see self.supported_official_language_plugin_names). If you are loading in a local plugin, the name can be anything as long as it does not conflict with an already loaded plugin name. @@ -187,43 +187,43 @@ def load(self, language_plugin_name: str, f".plugins.language_data.{language_plugin_name}.{language_plugin_name}", "wuggy").OfficialLanguagePlugin() - if language_plugin_name not in self.bigramchains: + if language_plugin_name not in self.segment_graphs: default_data_path = os.path.join( self.language_plugin_data_path, language_plugin.default_data) - self.bigramchains[self.language_plugin_name] = ( - self.__load_bigramchain(language_plugin, default_data_path)) + self.segment_graphs[self.language_plugin_name] = ( + self.__load_segment_graph(language_plugin, default_data_path)) self.__activate(self.language_plugin_name) @staticmethod - def __load_bigramchain(language_plugin, data_path: str) -> BigramChain: + def __load_segment_graph(language_plugin, data_path: str) -> SegmentGraph: """ - Load a bigram chain from its data file, going through a binary + Load a segment graph from its data file, going through a binary cache next to the data file so the text is parsed only once per data file version. Cache failures fall back to parsing silently. """ - cache_path = data_path + '.chain.pkl' + cache_path = data_path + '.graph.pkl' stat = os.stat(data_path) source_signature = (stat.st_mtime_ns, stat.st_size) try: with open(cache_path, 'rb') as f: cached = pickle.load(f) if cached.get('source') == source_signature: - return BigramChain.from_cache_state( + return SegmentGraph.from_cache_state( language_plugin, cached['state']) except (OSError, ValueError, KeyError, pickle.PickleError, EOFError, AttributeError): pass - bigramchain = BigramChain(language_plugin) + segment_graph = SegmentGraph(language_plugin) with open(data_path, 'r', encoding='utf-8') as data_file: - bigramchain.load(data_file) + segment_graph.load(data_file) try: with open(cache_path, 'wb') as f: pickle.dump({'source': source_signature, - 'state': bigramchain.to_cache_state()}, + 'state': segment_graph.to_cache_state()}, f, protocol=pickle.HIGHEST_PROTOCOL) except OSError: pass - return bigramchain + return segment_graph @staticmethod def remove_downloaded_language_plugins() -> None: @@ -297,14 +297,14 @@ def download_language_plugin( def __activate(self, name: str) -> None: """ - Activate a language plugin by setting the corresponding bigramchains and lexicon properties. + Activate a language plugin by setting the corresponding segment_graphs and lexicon properties. This deactivates and garbage collects any previously activated language plugin. Should only be called internally, do not call on your own. """ if isinstance(name, type(codecs)): name = name.__name__ - self.bigramchain = self.bigramchains[name] - self.language_plugin = self.bigramchain.language_plugin + self.segment_graph = self.segment_graphs[name] + self.language_plugin = self.segment_graph.language_plugin self.__load_neighbor_lexicon() self.__load_word_lexicon() self.__load_lookup_lexicon() @@ -397,7 +397,7 @@ def set_reference_sequence(self, sequence: str) -> None: """ self.reference_sequence = self.language_plugin.transform( sequence).representation - self.reference_sequence_frequencies = self.bigramchain.get_frequencies( + self.reference_sequence_frequencies = self.segment_graph.get_frequencies( self.reference_sequence) self.__clear_stat_cache() for name in self.__get_statistics(): @@ -502,7 +502,7 @@ def set_attribute_filter(self, name: str) -> None: raise ValueError( f"Attribute filter {name} is not supported.") self.attribute_filters[name] = reference_sequence - self.attribute_subchain = None + self.attribute_subgraph = None def set_attribute_filters(self, names: [str]) -> None: """ @@ -517,8 +517,8 @@ def __apply_attribute_filters(self) -> None: This is currently used by Wuggy internally, do not call on your own. """ for attribute, reference_sequence in self.attribute_filters.items(): - subchain = self.attribute_subchain if self.attribute_subchain is not None else self.bigramchain - self.attribute_subchain = subchain.attribute_filter( + subgraph = self.attribute_subgraph if self.attribute_subgraph is not None else self.segment_graph + self.attribute_subgraph = subgraph.attribute_filter( reference_sequence, attribute) def clear_attribute_filters(self) -> None: @@ -539,7 +539,7 @@ def clear_frequency_filter(self) -> None: Clear the previously set frequency filter. """ self.frequency_filter = None - self.frequency_subchain = None + self.frequency_subgraph = None def apply_frequency_filter(self) -> None: """ @@ -548,8 +548,8 @@ def apply_frequency_filter(self) -> None: if self.frequency_filter is None: raise Exception("No frequency filter was set") reference_sequence, lower, upper = self.frequency_filter - subchain = self.attribute_subchain if self.attribute_subchain is not None else self.bigramchain - self.frequency_subchain = subchain.frequency_filter( + subgraph = self.attribute_subgraph if self.attribute_subgraph is not None else self.segment_graph + self.frequency_subgraph = subgraph.frequency_filter( reference_sequence, lower, upper) @_loaded_language_plugin_required @@ -616,24 +616,24 @@ def __generate_classic_inner( f"Sequence {input_sequence} was not found in lexicon {self.current_language_plugin_name}") self.set_reference_sequence(input_sequence_segments) self.set_output_mode(output_mode) - subchain = self.bigramchain + subgraph = self.segment_graph starttime = time() pseudoword_matches = [] frequency_exponent = 1 if match_subsyllabic_segment_length: self.set_attribute_filter("segment_length") self.__apply_attribute_filters() - subchain = self.attribute_subchain + subgraph = self.attribute_subgraph while True: if concentric_search: self.set_frequency_filter( 2**frequency_exponent, 2**frequency_exponent) frequency_exponent += 1 self.apply_frequency_filter() - subchain = self.frequency_subchain - subchain = subchain.clean(len(self.reference_sequence) - 1) - subchain.set_startkeys(self.reference_sequence) - for sequence in subchain.generate(mode=search_mode): + subgraph = self.frequency_subgraph + subgraph = subgraph.prune(len(self.reference_sequence) - 1) + subgraph.set_start_vertices(self.reference_sequence) + for sequence in subgraph.generate(mode=search_mode): # Mandatory statistics before finding a suitable match self.clear_statistics() self.set_statistics(["overlap_ratio", "plain_length", "lexicality"]) @@ -712,24 +712,24 @@ def __generate_gui_inner( f"Sequence {input_sequence} was not found in lexicon {self.current_language_plugin_name}") self.set_reference_sequence(input_sequence_segments) self.set_output_mode(output_mode) - subchain = self.bigramchain + subgraph = self.segment_graph starttime = time() pseudoword_matches = [] frequency_exponent = 1 if match_subsyllabic_segment_length: self.set_attribute_filter("segment_length") self.__apply_attribute_filters() - subchain = self.attribute_subchain + subgraph = self.attribute_subgraph while True: if concentric_search: self.set_frequency_filter( 2**frequency_exponent, 2**frequency_exponent) frequency_exponent += 1 self.apply_frequency_filter() - subchain = self.frequency_subchain - subchain = subchain.clean(len(self.reference_sequence) - 1) - subchain.set_startkeys(self.reference_sequence) - for sequence in subchain.generate(mode=search_mode): + subgraph = self.frequency_subgraph + subgraph = subgraph.prune(len(self.reference_sequence) - 1) + subgraph.set_start_vertices(self.reference_sequence) + for sequence in subgraph.generate(mode=search_mode): self.clear_statistics() self.set_statistics(["overlap_ratio", "plain_length", "lexicality"]) if (time() - starttime) >= max_search_time: @@ -779,23 +779,23 @@ def generate_advanced(self, clear_cache: bool = True, self.__clear_sequence_cache() if self.output_mode is None: self.set_output_mode("plain") - if len(self.attribute_filters) == 0 and self.frequency_subchain is None: - subchain = self.bigramchain + if len(self.attribute_filters) == 0 and self.frequency_subgraph is None: + subgraph = self.segment_graph if len(self.attribute_filters) != 0: - if self.attribute_subchain is None: + if self.attribute_subgraph is None: self.__apply_attribute_filters() - subchain = self.attribute_subchain + subgraph = self.attribute_subgraph if self.frequency_filter is not None: self.apply_frequency_filter() - subchain = self.frequency_subchain + subgraph = self.frequency_subgraph if self.reference_sequence is not None: - subchain = subchain.clean(len(self.reference_sequence) - 1) - subchain.set_startkeys(self.reference_sequence) + subgraph = subgraph.prune(len(self.reference_sequence) - 1) + subgraph.set_start_vertices(self.reference_sequence) else: warn( "No reference sequence was set. Ignore this message if this was intentional.") - subchain.set_startkeys() - for sequence in subchain.generate(mode=mode): + subgraph.set_start_vertices() + for sequence in subgraph.generate(mode=mode): if self.language_plugin.output_plain(sequence) in self.sequence_cache: pass else: diff --git a/wuggy/plugins/baselanguageplugin.py b/wuggy/plugins/baselanguageplugin.py index 18eb954..94eedec 100644 --- a/wuggy/plugins/baselanguageplugin.py +++ b/wuggy/plugins/baselanguageplugin.py @@ -173,7 +173,7 @@ def statistic_ned1(self, generator, generated_sequence): @difference def statistic_transition_frequencies(self, generator, generated_sequence): - return generator.bigramchain.get_frequencies(generated_sequence) + return generator.segment_graph.get_frequencies(generated_sequence) def onsetnucleuscoda(self, orthographic_syllable, lang=None): self.oncpattern = lang.oncpattern diff --git a/wuggy/utilities/bigramchain.py b/wuggy/utilities/segmentgraph.py similarity index 69% rename from wuggy/utilities/bigramchain.py rename to wuggy/utilities/segmentgraph.py index 7570196..b6db16b 100644 --- a/wuggy/utilities/bigramchain.py +++ b/wuggy/utilities/segmentgraph.py @@ -3,23 +3,23 @@ import numpy as np -Link = namedtuple('Link', ['position', 'value']) +Vertex = namedtuple('Vertex', ['position', 'value']) _SENTINEL = object() # Bump when the cached on-disk representation changes shape. -CACHE_FORMAT_VERSION = 1 +CACHE_FORMAT_VERSION = 2 -class BigramChain: +class SegmentGraph: """ - A layered DAG over (position, segment) nodes storing the next possible + A layered DAG over (position, segment) vertices storing the next possible values, given a list of input sequences. Nodes are interned to integer ids; edges live in flat NumPy arrays (source, destination, frequency) sorted by source position. Filters - return lightweight views that share the node/edge store and differ only - in their boolean edge mask, so no filter ever copies the chain. + return lightweight views that share the vertex/edge store and differ only + in their boolean edge mask, so no filter ever copies the graph. """ def __init__(self, language_plugin, data=None, encoding='utf-8', @@ -29,12 +29,12 @@ def __init__(self, language_plugin, data=None, encoding='utf-8', self.hidden_sequence = self.language_plugin.hidden_sequence except AttributeError: self.hidden_sequence = False - self.startkeys = {} + self.start_vertices = {} self.limit_frequencies = {} - # Shared node/edge store; populated by load() or shared by views. - self.node_segments = [] - self.node_positions = np.empty(0, dtype=np.int32) - self.node_index = {} + # Shared vertex/edge store; populated by load() or shared by views. + self.vertex_segments = [] + self.vertex_positions = np.empty(0, dtype=np.int32) + self.vertex_index = {} self.edge_src = np.empty(0, dtype=np.int32) self.edge_dst = np.empty(0, dtype=np.int32) self.edge_freq = np.empty(0, dtype=np.float64) @@ -50,22 +50,22 @@ def __init__(self, language_plugin, data=None, encoding='utf-8', # Construction def load(self, datafile, size=100, cutoff=1, token=False): - node_index = {} - node_segments = [] - node_positions = [] + vertex_index = {} + vertex_segments = [] + vertex_positions = [] edge_weights = {} separator = self.language_plugin.separator transform = self.language_plugin.transform def intern(position, segment): key = (position, segment) - node_id = node_index.get(key) - if node_id is None: - node_id = len(node_segments) - node_index[key] = node_id - node_segments.append(segment) - node_positions.append(position) - return node_id + vertex_id = vertex_index.get(key) + if vertex_id is None: + vertex_id = len(vertex_segments) + vertex_index[key] = vertex_id + vertex_segments.append(segment) + vertex_positions.append(position) + return vertex_id for line in datafile: fields = line.strip('\n\t').split(separator) @@ -85,9 +85,9 @@ def intern(position, segment): previous = current datafile.close() - self.node_index = node_index - self.node_segments = node_segments - self.node_positions = np.array(node_positions, dtype=np.int32) + self.vertex_index = vertex_index + self.vertex_segments = vertex_segments + self.vertex_positions = np.array(vertex_positions, dtype=np.int32) nedges = len(edge_weights) src = np.empty(nedges, dtype=np.int32) dst = np.empty(nedges, dtype=np.int32) @@ -98,7 +98,7 @@ def intern(position, segment): freq[i] = f # Sort edges by source position so clean() can sweep positions # back-to-front over contiguous slices. - pos = self.node_positions[src] if nedges else np.empty(0, np.int32) + pos = self.vertex_positions[src] if nedges else np.empty(0, np.int32) order = np.argsort(pos, kind='stable') self.edge_src = src[order] self.edge_dst = dst[order] @@ -107,7 +107,7 @@ def intern(position, segment): self._build_position_slices() self.edge_mask = np.ones(nedges, dtype=bool) self._invalidate_caches() - self.set_startkeys() + self.set_start_vertices() def to_cache_state(self): """ @@ -117,8 +117,8 @@ def to_cache_state(self): """ return { 'version': CACHE_FORMAT_VERSION, - 'node_tuples': [tuple(segment) for segment in self.node_segments], - 'node_positions': self.node_positions, + 'vertex_tuples': [tuple(segment) for segment in self.vertex_segments], + 'vertex_positions': self.vertex_positions, 'edge_src': self.edge_src, 'edge_dst': self.edge_dst, 'edge_freq': self.edge_freq, @@ -129,29 +129,29 @@ def to_cache_state(self): def from_cache_state(cls, language_plugin, state): if state.get('version') != CACHE_FORMAT_VERSION: raise ValueError('incompatible cache format') - chain = cls(language_plugin) + graph = cls(language_plugin) nfields = len(language_plugin.Segment._fields) segments = [] - for values in state['node_tuples']: + for values in state['vertex_tuples']: segment_cls = (language_plugin.Segment if len(values) == nfields else language_plugin.SegmentH) segments.append(segment_cls(*values)) - chain.node_segments = segments - chain.node_positions = state['node_positions'] - chain.node_index = { - (int(position), segment): node_id - for node_id, (position, segment) - in enumerate(zip(chain.node_positions, segments))} - chain.edge_src = state['edge_src'] - chain.edge_dst = state['edge_dst'] - chain.edge_freq = state['edge_freq'] - chain.edge_pos = state['edge_pos'] - chain._build_position_slices() - chain.edge_mask = np.ones(len(chain.edge_src), dtype=bool) - chain._invalidate_caches() - chain.set_startkeys() - return chain + graph.vertex_segments = segments + graph.vertex_positions = state['vertex_positions'] + graph.vertex_index = { + (int(position), segment): vertex_id + for vertex_id, (position, segment) + in enumerate(zip(graph.vertex_positions, segments))} + graph.edge_src = state['edge_src'] + graph.edge_dst = state['edge_dst'] + graph.edge_freq = state['edge_freq'] + graph.edge_pos = state['edge_pos'] + graph._build_position_slices() + graph.edge_mask = np.ones(len(graph.edge_src), dtype=bool) + graph._invalidate_caches() + graph.set_start_vertices() + return graph def _build_position_slices(self): self._position_slices = {} @@ -164,15 +164,15 @@ def _build_position_slices(self): self._position_slices[int(position)] = slice(int(start), int(end)) def _view(self, edge_mask): - """A chain sharing this chain's store with a different edge mask.""" - view = BigramChain.__new__(BigramChain) + """A graph sharing this graph's store with a different edge mask.""" + view = SegmentGraph.__new__(SegmentGraph) view.language_plugin = self.language_plugin view.hidden_sequence = self.hidden_sequence - view.startkeys = {} + view.start_vertices = {} view.limit_frequencies = {} - view.node_segments = self.node_segments - view.node_positions = self.node_positions - view.node_index = self.node_index + view.vertex_segments = self.vertex_segments + view.vertex_positions = self.vertex_positions + view.vertex_index = self.vertex_index view.edge_src = self.edge_src view.edge_dst = self.edge_dst view.edge_freq = self.edge_freq @@ -208,8 +208,8 @@ def _adjacency(self): self._out_edges = dict(adjacency) return self._out_edges - def _node_id(self, position, segment): - return self.node_index.get((position, segment)) + def _vertex_id(self, position, segment): + return self.vertex_index.get((position, segment)) # ------------------------------------------------------------------ # Mapping-style compatibility (cold paths: display, dumps, limits) @@ -218,44 +218,44 @@ def __len__(self): return len(self._adjacency()) def __contains__(self, key): - node_id = self._node_id(key.position, key.value) - return node_id is not None and node_id in self._adjacency() + vertex_id = self._vertex_id(key.position, key.value) + return vertex_id is not None and vertex_id in self._adjacency() def keys(self): - return [Link(int(self.node_positions[node_id]), - self.node_segments[node_id]) - for node_id in self._adjacency()] + return [Vertex(int(self.vertex_positions[vertex_id]), + self.vertex_segments[vertex_id]) + for vertex_id in self._adjacency()] def items(self): - for node_id, children in self._adjacency().items(): - key = Link(int(self.node_positions[node_id]), - self.node_segments[node_id]) + for vertex_id, children in self._adjacency().items(): + key = Vertex(int(self.vertex_positions[vertex_id]), + self.vertex_segments[vertex_id]) edges = self._edges() nextkeys = { - Link(int(self.node_positions[child]), - self.node_segments[child]): - edges[(node_id, child)] + Vertex(int(self.vertex_positions[child]), + self.vertex_segments[child]): + edges[(vertex_id, child)] for child in children} yield key, nextkeys # ------------------------------------------------------------------ # Public interface - def set_startkeys(self, reference_sequence=None, fields=None): + def set_start_vertices(self, reference_sequence=None, fields=None): if fields is None: fields = self.language_plugin.default_fields adjacency = self._adjacency() - self.startkeys = { - Link(0, self.node_segments[node_id]): 0 - for node_id in adjacency - if self.node_positions[node_id] == 0} + self.start_vertices = { + Vertex(0, self.vertex_segments[vertex_id]): 0 + for vertex_id in adjacency + if self.vertex_positions[vertex_id] == 0} def get_frequencies(self, reference_sequence): frequencies = {} edges = self._edges() for position in range(len(reference_sequence) - 1): - source = self._node_id(position, reference_sequence[position]) - target = self._node_id( + source = self._vertex_id(position, reference_sequence[position]) + target = self._vertex_id( position + 1, reference_sequence[position + 1]) if source is None or target is None: frequencies[position] = 0 @@ -307,22 +307,22 @@ def frequency_filter(self, reference_sequence, lower, upper, kind='dev'): & (self.edge_freq <= edge_upper)) else: mask = self.edge_mask.copy() - result = self._view(mask)._clean_in_place(nref - 1) - result.set_startkeys() + result = self._view(mask)._prune_in_place(nref - 1) + result.set_start_vertices() return result def segmentset_filter(self, reference_sequence, segmentset): segmentset = segmentset.union(set(('^', '$'))) - node_pass = np.fromiter( - (segment.letters in segmentset for segment in self.node_segments), - dtype=bool, count=len(self.node_segments)) + vertex_pass = np.fromiter( + (segment.letters in segmentset for segment in self.vertex_segments), + dtype=bool, count=len(self.vertex_segments)) if len(self.edge_mask): mask = (self.edge_mask - & node_pass[self.edge_src] & node_pass[self.edge_dst]) + & vertex_pass[self.edge_src] & vertex_pass[self.edge_dst]) else: mask = self.edge_mask.copy() - result = self._view(mask)._clean_in_place(len(reference_sequence) - 1) - result.set_startkeys() + result = self._view(mask)._prune_in_place(len(reference_sequence) - 1) + result.set_start_vertices() return result def attribute_filter(self, reference_sequence, attribute): @@ -333,25 +333,25 @@ def reference_value(position): else: def reference_value(position): return reference_sequence[position] - node_pass = np.zeros(len(self.node_segments), dtype=bool) - for node_id, segment in enumerate(self.node_segments): - position = int(self.node_positions[node_id]) + vertex_pass = np.zeros(len(self.vertex_segments), dtype=bool) + for vertex_id, segment in enumerate(self.vertex_segments): + position = int(self.vertex_positions[vertex_id]) if position < nref: - node_pass[node_id] = ( + vertex_pass[vertex_id] = ( getattr(segment, attribute) == reference_value(position)) if len(self.edge_mask): - mask = self.edge_mask & node_pass[self.edge_src] + mask = self.edge_mask & vertex_pass[self.edge_src] else: mask = self.edge_mask.copy() return self._view(mask) - def clean(self, maxpos): + def prune(self, maxpos): """ - Remove chains that can not be completed. + Remove paths that can not be completed. """ - return self._view(self.edge_mask.copy())._clean_in_place(maxpos) + return self._view(self.edge_mask.copy())._prune_in_place(maxpos) - def _clean_in_place(self, maxpos): + def _prune_in_place(self, maxpos): """ Keep only edges on a path reaching position maxpos: one backward sweep over positions instead of the old copy-until-fixpoint pass. @@ -362,7 +362,7 @@ def _clean_in_place(self, maxpos): if len(mask) == 0: return self mask[self.edge_pos >= maxpos] = False - alive = self.node_positions == maxpos + alive = self.vertex_positions == maxpos for position in range(maxpos - 1, -1, -1): edge_slice = self._position_slices.get(position) if edge_slice is None: @@ -373,68 +373,68 @@ def _clean_in_place(self, maxpos): self._invalidate_caches() return self - def _resolve_startkeys(self, startkeys, adjacency): - """[(segment value, node id or None if not in the chain), ...]""" + def _resolve_start_vertices(self, start_vertices, adjacency): + """[(segment value, vertex id or None if not in the graph), ...]""" starts = [] - for key in startkeys: - node_id = self._node_id(key.position, key.value) - if node_id is not None and node_id in adjacency: - starts.append((key.value, node_id)) + for key in start_vertices: + vertex_id = self._vertex_id(key.position, key.value) + if vertex_id is not None and vertex_id in adjacency: + starts.append((key.value, vertex_id)) else: starts.append((key.value, None)) return starts def _completion_counts(self, weighted=False): """ - Per-node number of complete paths (or, when weighted, total path - weight as the product of edge frequencies) from the node to any - terminal node, over active edges. A node with no active outgoing + Per-vertex number of complete paths (or, when weighted, total path + weight as the product of edge frequencies) from the vertex to any + terminal vertex, over active edges. A vertex with no active outgoing edges is a terminal and counts as one path of weight 1, matching generate()'s yield condition. """ adjacency = self._adjacency() - nodes = set(adjacency) + vertices = set(adjacency) for children in adjacency.values(): - nodes.update(children) + vertices.update(children) edges = self._edges() if weighted else None counts = {} - for node in sorted( - nodes, key=lambda n: int(self.node_positions[n]), + for vertex in sorted( + vertices, key=lambda n: int(self.vertex_positions[n]), reverse=True): - children = adjacency.get(node) + children = adjacency.get(vertex) if not children: - counts[node] = 1.0 if weighted else 1 + counts[vertex] = 1.0 if weighted else 1 elif weighted: - counts[node] = sum( - edges[(node, child)] * counts[child] + counts[vertex] = sum( + edges[(vertex, child)] * counts[child] for child in children) else: - counts[node] = sum(counts[child] for child in children) + counts[vertex] = sum(counts[child] for child in children) return counts - def count_paths(self, startkeys=None): + def count_paths(self, start_vertices=None): """ Exact number of sequences generate() would yield, without - generating them. Startkeys that are not part of the chain yield a + generating them. Start vertices that are not part of the graph yield a single one-segment sequence and therefore count as one path. """ - if startkeys is None: - startkeys = self.startkeys + if start_vertices is None: + start_vertices = self.start_vertices counts = self._completion_counts() total = 0 - for _, node_id in self._resolve_startkeys(startkeys, + for _, vertex_id in self._resolve_start_vertices(start_vertices, self._adjacency()): - total += 1 if node_id is None else counts[node_id] + total += 1 if vertex_id is None else counts[vertex_id] return total - def generate(self, startkeys=None, mode='exhaustive', rng=random): + def generate(self, start_vertices=None, mode='exhaustive', rng=random): """ - Yield complete segment sequences from the (filtered) chain. + Yield complete segment sequences from the (filtered) graph. Modes: 'exhaustive' — depth-first traversal with shuffled edge order (the classic behavior). Complete, but sequence order is - biased toward low-branching regions of the chain. + biased toward low-branching regions of the graph. 'uniform' — every sequence exactly once, in unbiased random order (uniform sampling without replacement). 'weighted' — every sequence exactly once, sampled without @@ -443,22 +443,22 @@ def generate(self, startkeys=None, mode='exhaustive', rng=random): sequences tend to arrive first. """ if mode == 'uniform': - return self._generate_sampled(startkeys, weighted=False, rng=rng) + return self._generate_sampled(start_vertices, weighted=False, rng=rng) if mode == 'weighted': - return self._generate_sampled(startkeys, weighted=True, rng=rng) + return self._generate_sampled(start_vertices, weighted=True, rng=rng) if mode != 'exhaustive': raise ValueError(f"Unknown generation mode {mode!r}") - return self._generate_exhaustive(startkeys, rng) + return self._generate_exhaustive(start_vertices, rng) - def _generate_exhaustive(self, startkeys, rng): - if startkeys is None: - startkeys = self.startkeys + def _generate_exhaustive(self, start_vertices, rng): + if start_vertices is None: + start_vertices = self.start_vertices if len(self) == 0: - raise Exception('LinkError') + raise Exception('EmptyGraphError') adjacency = self._adjacency() - segments = self.node_segments + segments = self.vertex_segments - starts = self._resolve_startkeys(startkeys, adjacency) + starts = self._resolve_start_vertices(start_vertices, adjacency) rng.shuffle(starts) for start_value, start_id in starts: @@ -470,70 +470,70 @@ def _generate_exhaustive(self, startkeys, rng): rng.shuffle(children) stack = [iter(children)] while stack: - node_id = next(stack[-1], _SENTINEL) - if node_id is _SENTINEL: + vertex_id = next(stack[-1], _SENTINEL) + if vertex_id is _SENTINEL: stack.pop() path.pop() continue - children = adjacency.get(node_id) + children = adjacency.get(vertex_id) if children: - path.append(node_id) + path.append(vertex_id) children = children[:] rng.shuffle(children) stack.append(iter(children)) else: yield tuple(segments[i] for i in path) + ( - segments[node_id],) + segments[vertex_id],) - def _generate_sampled(self, startkeys, weighted, rng): + def _generate_sampled(self, start_vertices, weighted, rng): """ Sampling without replacement over complete paths. - The completion counts/weights below a node are shared by every + The completion counts/weights below a vertex are shared by every prefix that reaches it, so a drawn path must only be removed for its own prefix. Removed paths are therefore tracked in a prefix trie: during a draw, the available mass under child c is the - node's total (count or weight) minus what the trie records as + vertex's total (count or weight) minus what the trie records as already drawn through this exact prefix. Integer counts guarantee each path is yielded exactly once and the generator terminates; float weights only bias the selection order in weighted mode. Memory grows with the number of sequences drawn (one trie chain per draw); use exhaustive mode for memory-free full drains. """ - if startkeys is None: - startkeys = self.startkeys + if start_vertices is None: + start_vertices = self.start_vertices if len(self) == 0: - raise Exception('LinkError') + raise Exception('EmptyGraphError') adjacency = self._adjacency() - segments = self.node_segments + segments = self.vertex_segments edges = self._edges() counts = self._completion_counts() weights = self._completion_counts(weighted=True) if weighted else None - starts = self._resolve_startkeys(startkeys, adjacency) - # Trie node: [removed_count, removed_weight, children_by_dag_node]. + starts = self._resolve_start_vertices(start_vertices, adjacency) + # Trie vertex: [removed_count, removed_weight, children_by_dag_vertex]. trie_root = [0, 0.0, {}] singletons_drawn = set() - def available(trie_children, node_id): - removed = trie_children.get(node_id) - return counts[node_id] - (removed[0] if removed else 0) + def available(trie_children, vertex_id): + removed = trie_children.get(vertex_id) + return counts[vertex_id] - (removed[0] if removed else 0) while True: root_children = trie_root[2] live = [] - for index, (value, node_id) in enumerate(starts): - if node_id is None: + for index, (value, vertex_id) in enumerate(starts): + if vertex_id is None: if index not in singletons_drawn: live.append((index, value, None, 1, 1.0)) else: - avail = available(root_children, node_id) + avail = available(root_children, vertex_id) if avail > 0: - removed = root_children.get(node_id) - mass = ((weights[node_id] + removed = root_children.get(vertex_id) + mass = ((weights[vertex_id] - (removed[1] if removed else 0.0)) if weighted else float(avail)) - live.append((index, value, node_id, avail, mass)) + live.append((index, value, vertex_id, avail, mass)) if not live: return @@ -554,15 +554,15 @@ def available(trie_children, node_id): if pick < 0: chosen = entry break - index, value, node_id = chosen[0], chosen[1], chosen[2] - if node_id is None: + index, value, vertex_id = chosen[0], chosen[1], chosen[2] + if vertex_id is None: singletons_drawn.add(index) yield (value,) continue - path = [node_id] - current = node_id - trie_cursor = trie_root[2].get(node_id) + path = [vertex_id] + current = vertex_id + trie_cursor = trie_root[2].get(vertex_id) while True: children = adjacency.get(current) if not children: @@ -611,11 +611,11 @@ def available(trie_children, node_id): suffix_weights[i] = (edges[(path[i], path[i + 1])] * suffix_weights[i + 1]) cursor = trie_root - for node, suffix_weight in zip(path, suffix_weights): - child = cursor[2].get(node) + for vertex, suffix_weight in zip(path, suffix_weights): + child = cursor[2].get(vertex) if child is None: child = [0, 0.0, {}] - cursor[2][node] = child + cursor[2][vertex] = child child[0] += 1 child[1] += suffix_weight cursor = child From d37501cf596418fa7b98cf0ffa34b537c285387a Mon Sep 17 00:00:00 2001 From: Emmanuel Keuleers Date: Sat, 18 Jul 2026 21:35:32 +0200 Subject: [PATCH 5/6] Add design notes on the positional-graph abstraction Record the analysis that the 2.0 core is a domain-agnostic weighted positional DAG (acyclic WFA) with exact path counting and unbiased constrained sampling, its reusable applications, and its scope limits. Include a task prompt for extracting the language-free kernel out from under SegmentGraph as a verifiable, behavior-preserving refactor. Co-Authored-By: Claude Fable 5 --- .../design/extract-graph-kernel.prompt.md | 108 ++++++++++++++++ .../design/positional-graph-abstraction.md | 121 ++++++++++++++++++ 2 files changed, 229 insertions(+) create mode 100644 documentation/design/extract-graph-kernel.prompt.md create mode 100644 documentation/design/positional-graph-abstraction.md diff --git a/documentation/design/extract-graph-kernel.prompt.md b/documentation/design/extract-graph-kernel.prompt.md new file mode 100644 index 0000000..e896dbd --- /dev/null +++ b/documentation/design/extract-graph-kernel.prompt.md @@ -0,0 +1,108 @@ +# Task prompt: sketch the extraction of a domain-agnostic graph kernel + +Paste everything below the line into a new agent session, working from +the repository root on branch `claude/wuggy-2.0-reengineering`. + +--- + +You are working in the Wuggy repository (a multilingual pseudoword +generator) on the local, unpushed branch `claude/wuggy-2.0-reengineering`. +The 2.0 rewrite turned the core bigram store into an interned integer DAG +with NumPy edge arrays, boolean-mask filters, exact path counting, and +uniform/weighted sampling. That core is in +`wuggy/utilities/segmentgraph.py` (class `SegmentGraph`, vertex namedtuple +`Vertex`). Background on why this is worth doing is in +`documentation/design/positional-graph-abstraction.md` — read it first. + +## Goal + +Sketch — and, if it comes out clean, implement — the extraction of a +**domain-agnostic weighted positional-DAG kernel** out from under the +Wuggy-specific `SegmentGraph`, with **no change in observable behavior**. +The deliverable is primarily a design: the module boundary, the kernel's +public interface, and exactly which state and methods move versus stay. +Implementing the refactor is welcome if it is verifiable against the +existing harness, but a crisp proposal with a skeleton is an acceptable +stopping point — do not force a half-finished rewrite. + +## The intended seam + +The kernel (proposed name `PositionalGraph`, in +`wuggy/utilities/positionalgraph.py`) should own everything that has no +knowledge of language: + +- symbol interning and the `(position, symbol)` vertex identity, treating + each symbol payload as an **opaque hashable** — the kernel must not + reference `.letters`, `.hidden`, `Segment`, `SegmentH`, `separator`, + `transform`, or any `language_plugin`; +- the flat edge arrays (`edge_src/dst/freq/pos`), the `edge_mask`, the + position slices, and the view/mask machinery (`_view`, + `_invalidate_caches`, `_edges`, `_adjacency`); +- generic filter *primitives*: an edge-mask filter and a vertex-predicate + / vertex-pass-array filter, plus the numeric `frequency_filter` band + logic (which is already language-free); +- `prune` / `_prune_in_place`, `set_start_vertices`, `get_frequencies`, + `count_paths`, `_completion_counts`, and all of `generate` / + `_generate_exhaustive` / `_generate_sampled` (the three search modes); +- cache serialization of the array state, with the linguistic payload + reconstruction left to the subclass (see `to_cache_state` / + `from_cache_state`, which currently reach into `language_plugin.Segment` + — that reconstruction must move to the Wuggy layer). + +`SegmentGraph` then becomes a thin layer (subclass or composition — you +decide, argue the trade-off) that adds only the linguistic parts: + +- `load()` parsing a plugin data file via `language_plugin.separator` and + `language_plugin.transform`; +- `attribute_filter` and `segmentset_filter`, re-expressed in terms of the + kernel's generic vertex-predicate primitive (`segmentset_filter`'s use + of `segment.letters` is the clearest linguistic dependency; + `attribute_filter`'s `getattr(segment, attribute)` is nearly generic); +- `build_limit_frequencies` (uses named fields); +- the Segment/SegmentH reconstruction half of the cache round-trip. + +Watch for: the mapping-compat methods (`__len__`, `__contains__`, `keys`, +`items`, `display`) are used by cold paths and tests — keep them working, +and decide which layer they belong to. `WuggyGenerator` refers to +`segment_graph`/`segment_graphs` and calls `attribute_filter`, +`frequency_filter`, `prune`, `set_start_vertices`, `generate(mode=...)`, +`get_frequencies`, `count_paths`; that public surface must not change. + +## Hard constraints + +- **No behavioral change.** This is a pure refactor. +- Keep the existing public API of `SegmentGraph` and `WuggyGenerator` + intact (the rename to graph terminology already happened this branch; + do not revert or re-alias it). +- Python only (numpy + rapidfuzz are the runtime deps). No new deps. + +## How to verify + +There is an equivalence harness and a sampling test suite. Set up a venv +with `numpy` and `rapidfuzz`, then: + +1. `python tests/synthetic_plugin/make_data.py` (regenerates committed + synthetic data deterministically; should be a no-op). +2. Capture behavior on the current commit **before** refactoring: + `python tests/capture_behavior.py --out /tmp/before.json`. +3. Do the extraction. +4. `python tests/capture_behavior.py --out /tmp/after.json` and confirm + `diff /tmp/before.json /tmp/after.json` is empty — this proves edge + dumps (type and token weighting), start vertices, exhaustive + generation sets, and all statistics are byte-identical. +5. `python tests/test_sampling.py` must pass (path counts, set-equality + of all three generate modes, seeded determinism, weighted bias, + generator API integration). + +If you implement it, commit on the same branch with a message explaining +the seam; do not push. If you only sketch it, write the proposal to +`documentation/design/graph-kernel-extraction-plan.md` with the concrete +method/state partition and the proposed kernel interface. + +## Deliverable + +Either a verified refactor (both checks green) or a written extraction +plan precise enough to implement without rediscovering the seam — the +exact list of what moves, what stays, the kernel's public signature, and +how `SegmentGraph` re-expresses `attribute_filter` / `segmentset_filter` +on top of it. diff --git a/documentation/design/positional-graph-abstraction.md b/documentation/design/positional-graph-abstraction.md new file mode 100644 index 0000000..496a2cb --- /dev/null +++ b/documentation/design/positional-graph-abstraction.md @@ -0,0 +1,121 @@ +# The positional-graph abstraction + +*Design note, written during the 2.0 reengineering (branch +`claude/wuggy-2.0-reengineering`).* + +## Context + +The 2.0 rewrite replaced Wuggy's dict-of-dicts bigram store with an +interned integer DAG held in flat arrays +([`wuggy/utilities/segmentgraph.py`](../../wuggy/utilities/segmentgraph.py), +class `SegmentGraph`). Filters became boolean masks over the edge array, +`prune()` became a single backward-reachability sweep, and generation +gained exact path counting (`count_paths`) plus uniform and weighted +sampling-without-replacement modes, all driven by one backward dynamic +programming sweep over the layers. + +A consequence worth recording: almost none of that machinery is about +language. It is a general data structure that happens to be carrying +subsyllabic segments today. + +## What the structure actually is + +Strip away the segments and lexicons and it is: + +> **A weighted, layered DAG — equivalently an acyclic weighted +> finite-state automaton — over positional symbols, supporting +> composable hard-constraint masks, exact path counting, and unbiased +> sampling without replacement, all driven by one backward DP.** + +Each *position* is a layer; each vertex is a `(position, symbol)` pair; +edges connect consecutive layers with weights equal to co-occurrence +counts (or summed token frequency). Every attested and every generable +sequence is a path from a start vertex to a vertex in the final layer. + +## The four reusable primitives + +1. **Compact representation of a combinatorial set as paths.** A set of + sequences too large to materialize is held as a small graph. + +2. **Constrain without materializing.** Each filter is a boolean AND + over the edge array; the admissible set is narrowed by intersecting + masks, never by enumerating and rejecting. This is what beats + rejection sampling when the admissible set is a tiny fraction of the + whole. + +3. **Count exactly.** `count_paths` answers a #P-style question that is + tractable *because the graph is a DAG* — one DP sweep. The weighted + version of the same sweep is the **partition function** of the + positional distribution. + +4. **Sample uniformly or by weight, without replacement, exactly.** + +## Applications, sorted by how cleanly they map + +### Maps directly (same machinery, different alphabet) + +- **Template-matched generation in general.** Wuggy is one instance of + "generate items resembling a template, varying a controlled proportion + of elements, drawn from attested statistics." The same engine drives + procedural names, style-matched synthetic/fuzz test corpora, or + de-novo biological sequence candidates (peptides, oligos, SMILES + fragments) under positional-composition constraints and attested + transition frequencies. + +- **Wordlikeness / typicality scoring, not just generation.** Transition + frequencies are already computed as a statistic; exposed as a + normalized scorer (transition-product ÷ partition function) it is a + general "how typical is this string for this lexicon" function — + psycholinguistic norming, narrow-domain spell/OCR/ASR candidate + rescoring, structural anomaly detection (low weight = atypical). + +- **Design-space analytics for experiments.** "How many stimuli survive + these controls?" answered instantly, plus quantities the partition + function unlocks: **entropy of the constrained sub-language** and **KL + divergence between two constrained sub-languages** — e.g. how much a + frequency band or a segment restriction actually shrinks and reshapes + the space. + +### Maps with a modest extension + +- **Layered constraint satisfaction / solution counting.** Read each + position as a CSP variable and the masks as a constraint store: + crossword filling, positionally-constrained scheduling, configurable- + product (feature-model) enumeration. Uniform sampling of *solutions* + and exact solution-counting are cheap here for the same DAG reason. + +## Honest scope boundaries + +- The model is **first-order and acyclic.** Higher-order context means + vertices become `(position, k-gram)` — same machinery, more vertices. + True cycles (a non-positional automaton) break the layered backward + sweep and would need a general topological-order DP, or matrix methods + in the cyclic case. + +- The weighting is a **product of raw frequencies, not a smoothed + probability model.** Fine for ranking and relative comparison, but + unseen transitions are hard zeros; treat it as a partition function + over attested mass, not a calibrated language model. + +- The masks express **hard** constraints well; soft/weighted constraints + beyond the frequency weighting are not modeled. + +## Implication: the extraction seam + +The core already contains no linguistics. This argues for a clean seam +between: + +- a **domain-agnostic kernel** — call it a `PositionalGraph` / + weighted-DAG module — owning interning, edge arrays, masks, counting, + and the three sampling modes; and + +- the **Wuggy-specific layer** — segments, ONC syllabification, + lexicality, OLD20, plugin-driven parsing — sitting on top of it. + +That is also exactly the boundary worth compiling if the counting-and- +sampling kernel ever becomes a Rust / `extendr` core: the kernel is the +part worth reusing and the part worth making fast, and it is separable +from everything that makes Wuggy specifically about pseudowords. + +See [`extract-graph-kernel.prompt.md`](extract-graph-kernel.prompt.md) +for a task prompt that sketches this extraction. From ffe3227e95edb67c539b9410df1301907db058d0 Mon Sep 17 00:00:00 2001 From: Emmanuel Keuleers Date: Sat, 18 Jul 2026 21:45:06 +0200 Subject: [PATCH 6/6] Extract domain-agnostic PositionalGraph kernel from SegmentGraph SegmentGraph mixed two concerns: a generic weighted positional-DAG engine and Wuggy's linguistic layer. This splits them at the seam identified in documentation/design/positional-graph-abstraction.md, with no change in observable behavior. wuggy/utilities/positionalgraph.py now owns everything that has no knowledge of language, treating each symbol payload as an opaque hashable: vertex interning and (position, symbol) identity, the flat edge arrays and edge_mask with position slices, the shared-store view machinery (_view/_invalidate_caches/_edges/_adjacency), the mapping- compat methods and display, set_start_vertices, get_frequencies, frequency_filter, prune/_prune_in_place, count_paths and _completion_counts, and all three generate modes. It gains three generic filter primitives (edge_mask_filter, vertex_pass_filter, vertex_predicate_filter), a generic loader (load_sequences, taking (symbols, weight) pairs), and the array half of cache serialization (_cache_arrays/_restore_arrays). SegmentGraph subclasses the kernel and keeps only the linguistic parts: load() parsing plugin data files via separator/transform, attribute_filter and segmentset_filter re-expressed through vertex_predicate_filter, build_limit_frequencies, and the Segment/SegmentH namedtuple reconstruction half of the cache round-trip (format unchanged, version 2, so existing caches load). Subclassing was chosen over composition because every filter returns a new graph object created inside kernel methods; with composition each returned view would need re-wrapping plus a pile of forwarding methods, whereas the kernel's _view() instantiating type(self) and a single _share_domain_state() hook (carrying language_plugin, hidden_sequence, limit_frequencies onto views) gives the same result with no delegation layer. The kernel stores payloads as vertex_symbols; SegmentGraph keeps a vertex_segments property as a compat alias. Two deliberate near-noop cleanups: set_start_vertices drops its dead language_plugin.default_fields lookup (both parameters were already unused), and load() now feeds the kernel through a generator while preserving the exact per-line order of transform, cutoff, and random.randint calls. Verified: tests/capture_behavior.py output is byte-identical to the pre-refactor baseline both when parsing fresh and when loading from the pickle cache, and tests/test_sampling.py passes with identical seeded statistics. Co-Authored-By: Claude Fable 5 --- wuggy/utilities/positionalgraph.py | 576 +++++++++++++++++++++++++++ wuggy/utilities/segmentgraph.py | 614 +++-------------------------- 2 files changed, 636 insertions(+), 554 deletions(-) create mode 100644 wuggy/utilities/positionalgraph.py diff --git a/wuggy/utilities/positionalgraph.py b/wuggy/utilities/positionalgraph.py new file mode 100644 index 0000000..8b49f34 --- /dev/null +++ b/wuggy/utilities/positionalgraph.py @@ -0,0 +1,576 @@ +import random +from collections import defaultdict, namedtuple + +import numpy as np + +Vertex = namedtuple('Vertex', ['position', 'value']) + +_SENTINEL = object() + + +class PositionalGraph: + """ + A weighted, layered DAG over (position, symbol) vertices storing the next + possible values, given a list of input sequences. + + Symbols are opaque hashable payloads; this class knows nothing about what + they mean. Vertices are interned to integer ids; edges live in flat NumPy + arrays (source, destination, frequency) sorted by source position. + Filters return lightweight views that share the vertex/edge store and + differ only in their boolean edge mask, so no filter ever copies the + graph. On top of that store it supports composable hard-constraint masks + (`vertex_pass_filter`, `frequency_filter`), pruning to complete paths, + exact path counting, and exhaustive/uniform/weighted generation. + """ + + def __init__(self): + self.start_vertices = {} + # Shared vertex/edge store; populated by load_sequences() or shared + # by views. + self.vertex_symbols = [] + self.vertex_positions = np.empty(0, dtype=np.int32) + self.vertex_index = {} + self.edge_src = np.empty(0, dtype=np.int32) + self.edge_dst = np.empty(0, dtype=np.int32) + self.edge_freq = np.empty(0, dtype=np.float64) + self.edge_pos = np.empty(0, dtype=np.int32) + self._position_slices = {} + self.edge_mask = np.empty(0, dtype=bool) + self._edge_lookup = None + self._out_edges = None + + # ------------------------------------------------------------------ + # Construction + + def load_sequences(self, sequences): + """ + Build the store from an iterable of (symbols, weight) pairs, where + symbols is an indexable sequence of hashable payloads. Parallel + edges accumulate their weights. + """ + vertex_index = {} + vertex_symbols = [] + vertex_positions = [] + edge_weights = {} + + def intern(position, symbol): + key = (position, symbol) + vertex_id = vertex_index.get(key) + if vertex_id is None: + vertex_id = len(vertex_symbols) + vertex_index[key] = vertex_id + vertex_symbols.append(symbol) + vertex_positions.append(position) + return vertex_id + + for symbols, weight in sequences: + n = len(symbols) + previous = intern(0, symbols[0]) + for j in range(1, n): + current = intern(j, symbols[j]) + edge = (previous, current) + edge_weights[edge] = edge_weights.get(edge, 0) + weight + previous = current + + self.vertex_index = vertex_index + self.vertex_symbols = vertex_symbols + self.vertex_positions = np.array(vertex_positions, dtype=np.int32) + nedges = len(edge_weights) + src = np.empty(nedges, dtype=np.int32) + dst = np.empty(nedges, dtype=np.int32) + freq = np.empty(nedges, dtype=np.float64) + for i, ((s, d), f) in enumerate(edge_weights.items()): + src[i] = s + dst[i] = d + freq[i] = f + # Sort edges by source position so pruning can sweep positions + # back-to-front over contiguous slices. + pos = self.vertex_positions[src] if nedges else np.empty(0, np.int32) + order = np.argsort(pos, kind='stable') + self.edge_src = src[order] + self.edge_dst = dst[order] + self.edge_freq = freq[order] + self.edge_pos = pos[order] + self._build_position_slices() + self.edge_mask = np.ones(nedges, dtype=bool) + self._invalidate_caches() + self.set_start_vertices() + + def _cache_arrays(self): + """The array half of a picklable snapshot; symbols are the caller's.""" + return { + 'vertex_positions': self.vertex_positions, + 'edge_src': self.edge_src, + 'edge_dst': self.edge_dst, + 'edge_freq': self.edge_freq, + 'edge_pos': self.edge_pos, + } + + def _restore_arrays(self, state, vertex_symbols): + """Rebuild the store from _cache_arrays() output plus its symbols.""" + self.vertex_symbols = vertex_symbols + self.vertex_positions = state['vertex_positions'] + self.vertex_index = { + (int(position), symbol): vertex_id + for vertex_id, (position, symbol) + in enumerate(zip(self.vertex_positions, vertex_symbols))} + self.edge_src = state['edge_src'] + self.edge_dst = state['edge_dst'] + self.edge_freq = state['edge_freq'] + self.edge_pos = state['edge_pos'] + self._build_position_slices() + self.edge_mask = np.ones(len(self.edge_src), dtype=bool) + self._invalidate_caches() + self.set_start_vertices() + + def _build_position_slices(self): + self._position_slices = {} + if len(self.edge_pos) == 0: + return + positions = np.unique(self.edge_pos) + starts = np.searchsorted(self.edge_pos, positions, side='left') + ends = np.searchsorted(self.edge_pos, positions, side='right') + for position, start, end in zip(positions, starts, ends): + self._position_slices[int(position)] = slice(int(start), int(end)) + + def _view(self, edge_mask): + """A graph sharing this graph's store with a different edge mask.""" + view = type(self).__new__(type(self)) + view.start_vertices = {} + view.vertex_symbols = self.vertex_symbols + view.vertex_positions = self.vertex_positions + view.vertex_index = self.vertex_index + view.edge_src = self.edge_src + view.edge_dst = self.edge_dst + view.edge_freq = self.edge_freq + view.edge_pos = self.edge_pos + view._position_slices = self._position_slices + view.edge_mask = edge_mask + view._invalidate_caches() + self._share_domain_state(view) + return view + + def _share_domain_state(self, view): + """Hook for subclasses to carry their own state onto filter views.""" + + def _invalidate_caches(self): + self._edge_lookup = None + self._out_edges = None + + # ------------------------------------------------------------------ + # Derived structures (cached per view) + + def _edges(self): + """{(src_id, dst_id): frequency} over active edges.""" + if self._edge_lookup is None: + active = np.flatnonzero(self.edge_mask) + self._edge_lookup = { + (int(self.edge_src[i]), int(self.edge_dst[i])): + float(self.edge_freq[i]) + for i in active} + return self._edge_lookup + + def _adjacency(self): + """{src_id: [dst_id, ...]} over active edges.""" + if self._out_edges is None: + adjacency = defaultdict(list) + for i in np.flatnonzero(self.edge_mask): + adjacency[int(self.edge_src[i])].append(int(self.edge_dst[i])) + self._out_edges = dict(adjacency) + return self._out_edges + + def _vertex_id(self, position, symbol): + return self.vertex_index.get((position, symbol)) + + # ------------------------------------------------------------------ + # Mapping-style compatibility (cold paths: display, dumps, limits) + + def __len__(self): + return len(self._adjacency()) + + def __contains__(self, key): + vertex_id = self._vertex_id(key.position, key.value) + return vertex_id is not None and vertex_id in self._adjacency() + + def keys(self): + return [Vertex(int(self.vertex_positions[vertex_id]), + self.vertex_symbols[vertex_id]) + for vertex_id in self._adjacency()] + + def items(self): + for vertex_id, children in self._adjacency().items(): + key = Vertex(int(self.vertex_positions[vertex_id]), + self.vertex_symbols[vertex_id]) + edges = self._edges() + nextkeys = { + Vertex(int(self.vertex_positions[child]), + self.vertex_symbols[child]): + edges[(vertex_id, child)] + for child in children} + yield key, nextkeys + + # ------------------------------------------------------------------ + # Public interface + + def set_start_vertices(self, reference_sequence=None, fields=None): + adjacency = self._adjacency() + self.start_vertices = { + Vertex(0, self.vertex_symbols[vertex_id]): 0 + for vertex_id in adjacency + if self.vertex_positions[vertex_id] == 0} + + def get_frequencies(self, reference_sequence): + frequencies = {} + edges = self._edges() + for position in range(len(reference_sequence) - 1): + source = self._vertex_id(position, reference_sequence[position]) + target = self._vertex_id( + position + 1, reference_sequence[position + 1]) + if source is None or target is None: + frequencies[position] = 0 + else: + frequencies[position] = edges.get((source, target), 0) + return frequencies + + def edge_mask_filter(self, mask): + """View keeping only active edges that also pass the given mask.""" + if len(self.edge_mask): + mask = self.edge_mask & mask + else: + mask = self.edge_mask.copy() + return self._view(mask) + + def vertex_pass_filter(self, vertex_pass, require_dst=True): + """ + View keeping edges whose source vertex (and, unless require_dst is + False, destination vertex) passes the given boolean array. + """ + if len(self.edge_mask): + mask = self.edge_mask & vertex_pass[self.edge_src] + if require_dst: + mask = mask & vertex_pass[self.edge_dst] + else: + mask = self.edge_mask.copy() + return self._view(mask) + + def vertex_predicate_filter(self, predicate, require_dst=True): + """vertex_pass_filter with the array built from predicate(position, symbol).""" + vertex_pass = np.fromiter( + (predicate(int(position), symbol) + for position, symbol + in zip(self.vertex_positions, self.vertex_symbols)), + dtype=bool, count=len(self.vertex_symbols)) + return self.vertex_pass_filter(vertex_pass, require_dst=require_dst) + + def frequency_filter(self, reference_sequence, lower, upper, kind='dev'): + # Concentric search widens bands as 2**k without bound; clamp so the + # arbitrary-precision ints survive the conversion to float64. + lower = float(max(min(lower, 1e300), -1e300)) + upper = float(max(min(upper, 1e300), -1e300)) + nref = len(reference_sequence) + max_position = int(self.edge_pos[-1]) if len(self.edge_pos) else -1 + lower_by_position = np.full(max_position + 2, np.inf) + upper_by_position = np.full(max_position + 2, -np.inf) + if kind == 'dev': + frequencies = self.get_frequencies(reference_sequence) + for position, frequency in frequencies.items(): + if position <= max_position: + lower_by_position[position] = frequency - lower + upper_by_position[position] = frequency + upper + elif kind == 'limit': + limit = min(nref - 1, max_position + 1) + lower_by_position[:limit] = lower + upper_by_position[:limit] = upper + if len(self.edge_pos): + edge_lower = lower_by_position[self.edge_pos] + edge_upper = upper_by_position[self.edge_pos] + mask = (self.edge_mask + & (self.edge_freq >= edge_lower) + & (self.edge_freq <= edge_upper)) + else: + mask = self.edge_mask.copy() + result = self._view(mask)._prune_in_place(nref - 1) + result.set_start_vertices() + return result + + def prune(self, maxpos): + """ + Remove paths that can not be completed. + """ + return self._view(self.edge_mask.copy())._prune_in_place(maxpos) + + def _prune_in_place(self, maxpos): + """ + Keep only edges on a path reaching position maxpos: one backward + sweep over positions instead of the old copy-until-fixpoint pass. + An edge survives if its destination is at maxpos or its destination + has a surviving outgoing edge. + """ + mask = self.edge_mask + if len(mask) == 0: + return self + mask[self.edge_pos >= maxpos] = False + alive = self.vertex_positions == maxpos + for position in range(maxpos - 1, -1, -1): + edge_slice = self._position_slices.get(position) + if edge_slice is None: + continue + keep = mask[edge_slice] & alive[self.edge_dst[edge_slice]] + mask[edge_slice] = keep + alive[self.edge_src[edge_slice][keep]] = True + self._invalidate_caches() + return self + + def _resolve_start_vertices(self, start_vertices, adjacency): + """[(symbol value, vertex id or None if not in the graph), ...]""" + starts = [] + for key in start_vertices: + vertex_id = self._vertex_id(key.position, key.value) + if vertex_id is not None and vertex_id in adjacency: + starts.append((key.value, vertex_id)) + else: + starts.append((key.value, None)) + return starts + + def _completion_counts(self, weighted=False): + """ + Per-vertex number of complete paths (or, when weighted, total path + weight as the product of edge frequencies) from the vertex to any + terminal vertex, over active edges. A vertex with no active outgoing + edges is a terminal and counts as one path of weight 1, matching + generate()'s yield condition. + """ + adjacency = self._adjacency() + vertices = set(adjacency) + for children in adjacency.values(): + vertices.update(children) + edges = self._edges() if weighted else None + counts = {} + for vertex in sorted( + vertices, key=lambda n: int(self.vertex_positions[n]), + reverse=True): + children = adjacency.get(vertex) + if not children: + counts[vertex] = 1.0 if weighted else 1 + elif weighted: + counts[vertex] = sum( + edges[(vertex, child)] * counts[child] + for child in children) + else: + counts[vertex] = sum(counts[child] for child in children) + return counts + + def count_paths(self, start_vertices=None): + """ + Exact number of sequences generate() would yield, without + generating them. Start vertices that are not part of the graph yield a + single one-symbol sequence and therefore count as one path. + """ + if start_vertices is None: + start_vertices = self.start_vertices + counts = self._completion_counts() + total = 0 + for _, vertex_id in self._resolve_start_vertices(start_vertices, + self._adjacency()): + total += 1 if vertex_id is None else counts[vertex_id] + return total + + def generate(self, start_vertices=None, mode='exhaustive', rng=random): + """ + Yield complete symbol sequences from the (filtered) graph. + + Modes: + 'exhaustive' — depth-first traversal with shuffled edge order + (the classic behavior). Complete, but sequence order is + biased toward low-branching regions of the graph. + 'uniform' — every sequence exactly once, in unbiased random + order (uniform sampling without replacement). + 'weighted' — every sequence exactly once, sampled without + replacement with probability proportional to the product + of its transition frequencies, so the highest-weight + sequences tend to arrive first. + """ + if mode == 'uniform': + return self._generate_sampled(start_vertices, weighted=False, rng=rng) + if mode == 'weighted': + return self._generate_sampled(start_vertices, weighted=True, rng=rng) + if mode != 'exhaustive': + raise ValueError(f"Unknown generation mode {mode!r}") + return self._generate_exhaustive(start_vertices, rng) + + def _generate_exhaustive(self, start_vertices, rng): + if start_vertices is None: + start_vertices = self.start_vertices + if len(self) == 0: + raise Exception('EmptyGraphError') + adjacency = self._adjacency() + symbols = self.vertex_symbols + + starts = self._resolve_start_vertices(start_vertices, adjacency) + rng.shuffle(starts) + + for start_value, start_id in starts: + if start_id is None: + yield (start_value,) + continue + path = [start_id] + children = adjacency[start_id][:] + rng.shuffle(children) + stack = [iter(children)] + while stack: + vertex_id = next(stack[-1], _SENTINEL) + if vertex_id is _SENTINEL: + stack.pop() + path.pop() + continue + children = adjacency.get(vertex_id) + if children: + path.append(vertex_id) + children = children[:] + rng.shuffle(children) + stack.append(iter(children)) + else: + yield tuple(symbols[i] for i in path) + ( + symbols[vertex_id],) + + def _generate_sampled(self, start_vertices, weighted, rng): + """ + Sampling without replacement over complete paths. + + The completion counts/weights below a vertex are shared by every + prefix that reaches it, so a drawn path must only be removed for + its own prefix. Removed paths are therefore tracked in a prefix + trie: during a draw, the available mass under child c is the + vertex's total (count or weight) minus what the trie records as + already drawn through this exact prefix. Integer counts guarantee + each path is yielded exactly once and the generator terminates; + float weights only bias the selection order in weighted mode. + Memory grows with the number of sequences drawn (one trie chain + per draw); use exhaustive mode for memory-free full drains. + """ + if start_vertices is None: + start_vertices = self.start_vertices + if len(self) == 0: + raise Exception('EmptyGraphError') + adjacency = self._adjacency() + symbols = self.vertex_symbols + edges = self._edges() + counts = self._completion_counts() + weights = self._completion_counts(weighted=True) if weighted else None + + starts = self._resolve_start_vertices(start_vertices, adjacency) + # Trie vertex: [removed_count, removed_weight, children_by_dag_vertex]. + trie_root = [0, 0.0, {}] + singletons_drawn = set() + + def available(trie_children, vertex_id): + removed = trie_children.get(vertex_id) + return counts[vertex_id] - (removed[0] if removed else 0) + + while True: + root_children = trie_root[2] + live = [] + for index, (value, vertex_id) in enumerate(starts): + if vertex_id is None: + if index not in singletons_drawn: + live.append((index, value, None, 1, 1.0)) + else: + avail = available(root_children, vertex_id) + if avail > 0: + removed = root_children.get(vertex_id) + mass = ((weights[vertex_id] + - (removed[1] if removed else 0.0)) + if weighted else float(avail)) + live.append((index, value, vertex_id, avail, mass)) + if not live: + return + + if weighted: + total_mass = sum(entry[4] for entry in live) + pick = rng.random() * max(total_mass, 0.0) + chosen = live[-1] + for entry in live: + pick -= entry[4] + if pick <= 0: + chosen = entry + break + else: + pick = rng.randrange(sum(entry[3] for entry in live)) + chosen = live[-1] + for entry in live: + pick -= entry[3] + if pick < 0: + chosen = entry + break + index, value, vertex_id = chosen[0], chosen[1], chosen[2] + if vertex_id is None: + singletons_drawn.add(index) + yield (value,) + continue + + path = [vertex_id] + current = vertex_id + trie_cursor = trie_root[2].get(vertex_id) + while True: + children = adjacency.get(current) + if not children: + break + trie_children = trie_cursor[2] if trie_cursor else {} + live_children = [] + for child in children: + avail = available(trie_children, child) + if avail <= 0: + continue + if weighted: + removed = trie_children.get(child) + mass = edges[(current, child)] * ( + weights[child] + - (removed[1] if removed else 0.0)) + live_children.append((child, avail, mass)) + else: + live_children.append((child, avail, float(avail))) + if weighted: + total_mass = sum(mass for _, _, mass in live_children) + pick = rng.random() * max(total_mass, 0.0) + current = live_children[-1][0] + for child, _, mass in live_children: + pick -= mass + if pick <= 0: + current = child + break + else: + pick = rng.randrange( + sum(avail for _, avail, _ in live_children)) + current = live_children[-1][0] + for child, avail, _ in live_children: + pick -= avail + if pick < 0: + current = child + break + path.append(current) + trie_cursor = (trie_cursor[2].get(current) + if trie_cursor else None) + + # Record the drawn path in the trie: below position i the + # removed suffix weight is the product of the remaining edge + # frequencies (1 at the terminal). + suffix_weights = [1.0] * len(path) + for i in range(len(path) - 2, -1, -1): + suffix_weights[i] = (edges[(path[i], path[i + 1])] + * suffix_weights[i + 1]) + cursor = trie_root + for vertex, suffix_weight in zip(path, suffix_weights): + child = cursor[2].get(vertex) + if child is None: + child = [0, 0.0, {}] + cursor[2][vertex] = child + child[0] += 1 + child[1] += suffix_weight + cursor = child + yield tuple(symbols[i] for i in path) + + def display(self): + for key, nextkeys in sorted(self.items(), key=lambda x: x): + print('***', key.position, key.value) + for nextkey, frequency in nextkeys.items(): + print(nextkey.value, frequency) diff --git a/wuggy/utilities/segmentgraph.py b/wuggy/utilities/segmentgraph.py index b6db16b..5529348 100644 --- a/wuggy/utilities/segmentgraph.py +++ b/wuggy/utilities/segmentgraph.py @@ -1,129 +1,80 @@ import random -from collections import defaultdict, namedtuple +from collections import defaultdict -import numpy as np +from .positionalgraph import PositionalGraph, Vertex -Vertex = namedtuple('Vertex', ['position', 'value']) - -_SENTINEL = object() +__all__ = ['SegmentGraph', 'Vertex', 'CACHE_FORMAT_VERSION'] # Bump when the cached on-disk representation changes shape. CACHE_FORMAT_VERSION = 2 -class SegmentGraph: +class SegmentGraph(PositionalGraph): """ - A layered DAG over (position, segment) vertices storing the next possible - values, given a list of input sequences. - - Nodes are interned to integer ids; edges live in flat NumPy arrays - (source, destination, frequency) sorted by source position. Filters - return lightweight views that share the vertex/edge store and differ only - in their boolean edge mask, so no filter ever copies the graph. + A PositionalGraph whose symbols are linguistic segments from a language + plugin. This layer owns everything that knows about language: parsing a + plugin data file, filters expressed over segment fields, limit + frequencies over named fields, and reconstructing plugin namedtuple + segments when a cached graph is loaded. All graph mechanics — interning, + edge arrays, masks, pruning, counting, generation — live in the kernel. """ def __init__(self, language_plugin, data=None, encoding='utf-8', size=100, cutoff=1, token=False): + super().__init__() self.language_plugin = language_plugin try: self.hidden_sequence = self.language_plugin.hidden_sequence except AttributeError: self.hidden_sequence = False - self.start_vertices = {} self.limit_frequencies = {} - # Shared vertex/edge store; populated by load() or shared by views. - self.vertex_segments = [] - self.vertex_positions = np.empty(0, dtype=np.int32) - self.vertex_index = {} - self.edge_src = np.empty(0, dtype=np.int32) - self.edge_dst = np.empty(0, dtype=np.int32) - self.edge_freq = np.empty(0, dtype=np.float64) - self.edge_pos = np.empty(0, dtype=np.int32) - self._position_slices = {} - self.edge_mask = np.empty(0, dtype=bool) - self._edge_lookup = None - self._out_edges = None if data is not None: self.load(data, size=size, cutoff=cutoff, token=token) + # The kernel stores payloads as vertex_symbols; keep the segment-named + # accessor for callers of the pre-extraction attribute. + @property + def vertex_segments(self): + return self.vertex_symbols + + def _share_domain_state(self, view): + view.language_plugin = self.language_plugin + view.hidden_sequence = self.hidden_sequence + view.limit_frequencies = {} + # ------------------------------------------------------------------ # Construction def load(self, datafile, size=100, cutoff=1, token=False): - vertex_index = {} - vertex_segments = [] - vertex_positions = [] - edge_weights = {} separator = self.language_plugin.separator transform = self.language_plugin.transform - def intern(position, segment): - key = (position, segment) - vertex_id = vertex_index.get(key) - if vertex_id is None: - vertex_id = len(vertex_segments) - vertex_index[key] = vertex_id - vertex_segments.append(segment) - vertex_positions.append(position) - return vertex_id - - for line in datafile: - fields = line.strip('\n\t').split(separator) - reference, input_sequence, frequency = fields - frequency = float(frequency) if token == True else 1 - sequence = transform(input_sequence, frequency) - representation = sequence.representation - n = len(representation) - if frequency >= cutoff and ( - size >= 100 or random.randint(1, 100) <= size): - previous = intern(0, representation[0]) - for j in range(1, n): - current = intern(j, representation[j]) - edge = (previous, current) - edge_weights[edge] = edge_weights.get( - edge, 0) + sequence.frequency - previous = current + def parsed_sequences(): + for line in datafile: + fields = line.strip('\n\t').split(separator) + reference, input_sequence, frequency = fields + frequency = float(frequency) if token == True else 1 + sequence = transform(input_sequence, frequency) + if frequency >= cutoff and ( + size >= 100 or random.randint(1, 100) <= size): + yield sequence.representation, sequence.frequency + + self.load_sequences(parsed_sequences()) datafile.close() - self.vertex_index = vertex_index - self.vertex_segments = vertex_segments - self.vertex_positions = np.array(vertex_positions, dtype=np.int32) - nedges = len(edge_weights) - src = np.empty(nedges, dtype=np.int32) - dst = np.empty(nedges, dtype=np.int32) - freq = np.empty(nedges, dtype=np.float64) - for i, ((s, d), f) in enumerate(edge_weights.items()): - src[i] = s - dst[i] = d - freq[i] = f - # Sort edges by source position so clean() can sweep positions - # back-to-front over contiguous slices. - pos = self.vertex_positions[src] if nedges else np.empty(0, np.int32) - order = np.argsort(pos, kind='stable') - self.edge_src = src[order] - self.edge_dst = dst[order] - self.edge_freq = freq[order] - self.edge_pos = pos[order] - self._build_position_slices() - self.edge_mask = np.ones(nedges, dtype=bool) - self._invalidate_caches() - self.set_start_vertices() - def to_cache_state(self): """ Picklable snapshot of the parsed store. Segments are flattened to plain tuples because plugin namedtuple classes are class attributes and cannot be pickled by reference. """ - return { + state = { 'version': CACHE_FORMAT_VERSION, - 'vertex_tuples': [tuple(segment) for segment in self.vertex_segments], - 'vertex_positions': self.vertex_positions, - 'edge_src': self.edge_src, - 'edge_dst': self.edge_dst, - 'edge_freq': self.edge_freq, - 'edge_pos': self.edge_pos, + 'vertex_tuples': [tuple(segment) + for segment in self.vertex_symbols], } + state.update(self._cache_arrays()) + return state @classmethod def from_cache_state(cls, language_plugin, state): @@ -137,131 +88,33 @@ def from_cache_state(cls, language_plugin, state): if len(values) == nfields else language_plugin.SegmentH) segments.append(segment_cls(*values)) - graph.vertex_segments = segments - graph.vertex_positions = state['vertex_positions'] - graph.vertex_index = { - (int(position), segment): vertex_id - for vertex_id, (position, segment) - in enumerate(zip(graph.vertex_positions, segments))} - graph.edge_src = state['edge_src'] - graph.edge_dst = state['edge_dst'] - graph.edge_freq = state['edge_freq'] - graph.edge_pos = state['edge_pos'] - graph._build_position_slices() - graph.edge_mask = np.ones(len(graph.edge_src), dtype=bool) - graph._invalidate_caches() - graph.set_start_vertices() + graph._restore_arrays(state, segments) return graph - def _build_position_slices(self): - self._position_slices = {} - if len(self.edge_pos) == 0: - return - positions = np.unique(self.edge_pos) - starts = np.searchsorted(self.edge_pos, positions, side='left') - ends = np.searchsorted(self.edge_pos, positions, side='right') - for position, start, end in zip(positions, starts, ends): - self._position_slices[int(position)] = slice(int(start), int(end)) - - def _view(self, edge_mask): - """A graph sharing this graph's store with a different edge mask.""" - view = SegmentGraph.__new__(SegmentGraph) - view.language_plugin = self.language_plugin - view.hidden_sequence = self.hidden_sequence - view.start_vertices = {} - view.limit_frequencies = {} - view.vertex_segments = self.vertex_segments - view.vertex_positions = self.vertex_positions - view.vertex_index = self.vertex_index - view.edge_src = self.edge_src - view.edge_dst = self.edge_dst - view.edge_freq = self.edge_freq - view.edge_pos = self.edge_pos - view._position_slices = self._position_slices - view.edge_mask = edge_mask - view._invalidate_caches() - return view - - def _invalidate_caches(self): - self._edge_lookup = None - self._out_edges = None - - # ------------------------------------------------------------------ - # Derived structures (cached per view) - - def _edges(self): - """{(src_id, dst_id): frequency} over active edges.""" - if self._edge_lookup is None: - active = np.flatnonzero(self.edge_mask) - self._edge_lookup = { - (int(self.edge_src[i]), int(self.edge_dst[i])): - float(self.edge_freq[i]) - for i in active} - return self._edge_lookup - - def _adjacency(self): - """{src_id: [dst_id, ...]} over active edges.""" - if self._out_edges is None: - adjacency = defaultdict(list) - for i in np.flatnonzero(self.edge_mask): - adjacency[int(self.edge_src[i])].append(int(self.edge_dst[i])) - self._out_edges = dict(adjacency) - return self._out_edges - - def _vertex_id(self, position, segment): - return self.vertex_index.get((position, segment)) - - # ------------------------------------------------------------------ - # Mapping-style compatibility (cold paths: display, dumps, limits) - - def __len__(self): - return len(self._adjacency()) - - def __contains__(self, key): - vertex_id = self._vertex_id(key.position, key.value) - return vertex_id is not None and vertex_id in self._adjacency() - - def keys(self): - return [Vertex(int(self.vertex_positions[vertex_id]), - self.vertex_segments[vertex_id]) - for vertex_id in self._adjacency()] - - def items(self): - for vertex_id, children in self._adjacency().items(): - key = Vertex(int(self.vertex_positions[vertex_id]), - self.vertex_segments[vertex_id]) - edges = self._edges() - nextkeys = { - Vertex(int(self.vertex_positions[child]), - self.vertex_segments[child]): - edges[(vertex_id, child)] - for child in children} - yield key, nextkeys - # ------------------------------------------------------------------ - # Public interface + # Segment-aware filters and limits - def set_start_vertices(self, reference_sequence=None, fields=None): - if fields is None: - fields = self.language_plugin.default_fields - adjacency = self._adjacency() - self.start_vertices = { - Vertex(0, self.vertex_segments[vertex_id]): 0 - for vertex_id in adjacency - if self.vertex_positions[vertex_id] == 0} + def segmentset_filter(self, reference_sequence, segmentset): + segmentset = segmentset.union(set(('^', '$'))) + result = self.vertex_predicate_filter( + lambda position, segment: segment.letters in segmentset) + result._prune_in_place(len(reference_sequence) - 1) + result.set_start_vertices() + return result - def get_frequencies(self, reference_sequence): - frequencies = {} - edges = self._edges() - for position in range(len(reference_sequence) - 1): - source = self._vertex_id(position, reference_sequence[position]) - target = self._vertex_id( - position + 1, reference_sequence[position + 1]) - if source is None or target is None: - frequencies[position] = 0 - else: - frequencies[position] = edges.get((source, target), 0) - return frequencies + def attribute_filter(self, reference_sequence, attribute): + nref = len(reference_sequence) + if type(reference_sequence[0]) == self.language_plugin.Segment: + def reference_value(position): + return getattr(reference_sequence[position], attribute) + else: + def reference_value(position): + return reference_sequence[position] + return self.vertex_predicate_filter( + lambda position, segment: ( + position < nref + and getattr(segment, attribute) == reference_value(position)), + require_dst=False) def build_limit_frequencies(self, fields): limits = defaultdict(dict) @@ -279,350 +132,3 @@ def build_limit_frequencies(self, fields): maxfrequency = limits[subkey].get('max', frequency) limits[subkey]['max'] = max(maxfrequency, frequency) self.limit_frequencies[tuple(fields)] = limits - - def frequency_filter(self, reference_sequence, lower, upper, kind='dev'): - # Concentric search widens bands as 2**k without bound; clamp so the - # arbitrary-precision ints survive the conversion to float64. - lower = float(max(min(lower, 1e300), -1e300)) - upper = float(max(min(upper, 1e300), -1e300)) - nref = len(reference_sequence) - max_position = int(self.edge_pos[-1]) if len(self.edge_pos) else -1 - lower_by_position = np.full(max_position + 2, np.inf) - upper_by_position = np.full(max_position + 2, -np.inf) - if kind == 'dev': - frequencies = self.get_frequencies(reference_sequence) - for position, frequency in frequencies.items(): - if position <= max_position: - lower_by_position[position] = frequency - lower - upper_by_position[position] = frequency + upper - elif kind == 'limit': - limit = min(nref - 1, max_position + 1) - lower_by_position[:limit] = lower - upper_by_position[:limit] = upper - if len(self.edge_pos): - edge_lower = lower_by_position[self.edge_pos] - edge_upper = upper_by_position[self.edge_pos] - mask = (self.edge_mask - & (self.edge_freq >= edge_lower) - & (self.edge_freq <= edge_upper)) - else: - mask = self.edge_mask.copy() - result = self._view(mask)._prune_in_place(nref - 1) - result.set_start_vertices() - return result - - def segmentset_filter(self, reference_sequence, segmentset): - segmentset = segmentset.union(set(('^', '$'))) - vertex_pass = np.fromiter( - (segment.letters in segmentset for segment in self.vertex_segments), - dtype=bool, count=len(self.vertex_segments)) - if len(self.edge_mask): - mask = (self.edge_mask - & vertex_pass[self.edge_src] & vertex_pass[self.edge_dst]) - else: - mask = self.edge_mask.copy() - result = self._view(mask)._prune_in_place(len(reference_sequence) - 1) - result.set_start_vertices() - return result - - def attribute_filter(self, reference_sequence, attribute): - nref = len(reference_sequence) - if type(reference_sequence[0]) == self.language_plugin.Segment: - def reference_value(position): - return getattr(reference_sequence[position], attribute) - else: - def reference_value(position): - return reference_sequence[position] - vertex_pass = np.zeros(len(self.vertex_segments), dtype=bool) - for vertex_id, segment in enumerate(self.vertex_segments): - position = int(self.vertex_positions[vertex_id]) - if position < nref: - vertex_pass[vertex_id] = ( - getattr(segment, attribute) == reference_value(position)) - if len(self.edge_mask): - mask = self.edge_mask & vertex_pass[self.edge_src] - else: - mask = self.edge_mask.copy() - return self._view(mask) - - def prune(self, maxpos): - """ - Remove paths that can not be completed. - """ - return self._view(self.edge_mask.copy())._prune_in_place(maxpos) - - def _prune_in_place(self, maxpos): - """ - Keep only edges on a path reaching position maxpos: one backward - sweep over positions instead of the old copy-until-fixpoint pass. - An edge survives if its destination is at maxpos or its destination - has a surviving outgoing edge. - """ - mask = self.edge_mask - if len(mask) == 0: - return self - mask[self.edge_pos >= maxpos] = False - alive = self.vertex_positions == maxpos - for position in range(maxpos - 1, -1, -1): - edge_slice = self._position_slices.get(position) - if edge_slice is None: - continue - keep = mask[edge_slice] & alive[self.edge_dst[edge_slice]] - mask[edge_slice] = keep - alive[self.edge_src[edge_slice][keep]] = True - self._invalidate_caches() - return self - - def _resolve_start_vertices(self, start_vertices, adjacency): - """[(segment value, vertex id or None if not in the graph), ...]""" - starts = [] - for key in start_vertices: - vertex_id = self._vertex_id(key.position, key.value) - if vertex_id is not None and vertex_id in adjacency: - starts.append((key.value, vertex_id)) - else: - starts.append((key.value, None)) - return starts - - def _completion_counts(self, weighted=False): - """ - Per-vertex number of complete paths (or, when weighted, total path - weight as the product of edge frequencies) from the vertex to any - terminal vertex, over active edges. A vertex with no active outgoing - edges is a terminal and counts as one path of weight 1, matching - generate()'s yield condition. - """ - adjacency = self._adjacency() - vertices = set(adjacency) - for children in adjacency.values(): - vertices.update(children) - edges = self._edges() if weighted else None - counts = {} - for vertex in sorted( - vertices, key=lambda n: int(self.vertex_positions[n]), - reverse=True): - children = adjacency.get(vertex) - if not children: - counts[vertex] = 1.0 if weighted else 1 - elif weighted: - counts[vertex] = sum( - edges[(vertex, child)] * counts[child] - for child in children) - else: - counts[vertex] = sum(counts[child] for child in children) - return counts - - def count_paths(self, start_vertices=None): - """ - Exact number of sequences generate() would yield, without - generating them. Start vertices that are not part of the graph yield a - single one-segment sequence and therefore count as one path. - """ - if start_vertices is None: - start_vertices = self.start_vertices - counts = self._completion_counts() - total = 0 - for _, vertex_id in self._resolve_start_vertices(start_vertices, - self._adjacency()): - total += 1 if vertex_id is None else counts[vertex_id] - return total - - def generate(self, start_vertices=None, mode='exhaustive', rng=random): - """ - Yield complete segment sequences from the (filtered) graph. - - Modes: - 'exhaustive' — depth-first traversal with shuffled edge order - (the classic behavior). Complete, but sequence order is - biased toward low-branching regions of the graph. - 'uniform' — every sequence exactly once, in unbiased random - order (uniform sampling without replacement). - 'weighted' — every sequence exactly once, sampled without - replacement with probability proportional to the product - of its transition frequencies, so the most word-like - sequences tend to arrive first. - """ - if mode == 'uniform': - return self._generate_sampled(start_vertices, weighted=False, rng=rng) - if mode == 'weighted': - return self._generate_sampled(start_vertices, weighted=True, rng=rng) - if mode != 'exhaustive': - raise ValueError(f"Unknown generation mode {mode!r}") - return self._generate_exhaustive(start_vertices, rng) - - def _generate_exhaustive(self, start_vertices, rng): - if start_vertices is None: - start_vertices = self.start_vertices - if len(self) == 0: - raise Exception('EmptyGraphError') - adjacency = self._adjacency() - segments = self.vertex_segments - - starts = self._resolve_start_vertices(start_vertices, adjacency) - rng.shuffle(starts) - - for start_value, start_id in starts: - if start_id is None: - yield (start_value,) - continue - path = [start_id] - children = adjacency[start_id][:] - rng.shuffle(children) - stack = [iter(children)] - while stack: - vertex_id = next(stack[-1], _SENTINEL) - if vertex_id is _SENTINEL: - stack.pop() - path.pop() - continue - children = adjacency.get(vertex_id) - if children: - path.append(vertex_id) - children = children[:] - rng.shuffle(children) - stack.append(iter(children)) - else: - yield tuple(segments[i] for i in path) + ( - segments[vertex_id],) - - def _generate_sampled(self, start_vertices, weighted, rng): - """ - Sampling without replacement over complete paths. - - The completion counts/weights below a vertex are shared by every - prefix that reaches it, so a drawn path must only be removed for - its own prefix. Removed paths are therefore tracked in a prefix - trie: during a draw, the available mass under child c is the - vertex's total (count or weight) minus what the trie records as - already drawn through this exact prefix. Integer counts guarantee - each path is yielded exactly once and the generator terminates; - float weights only bias the selection order in weighted mode. - Memory grows with the number of sequences drawn (one trie chain - per draw); use exhaustive mode for memory-free full drains. - """ - if start_vertices is None: - start_vertices = self.start_vertices - if len(self) == 0: - raise Exception('EmptyGraphError') - adjacency = self._adjacency() - segments = self.vertex_segments - edges = self._edges() - counts = self._completion_counts() - weights = self._completion_counts(weighted=True) if weighted else None - - starts = self._resolve_start_vertices(start_vertices, adjacency) - # Trie vertex: [removed_count, removed_weight, children_by_dag_vertex]. - trie_root = [0, 0.0, {}] - singletons_drawn = set() - - def available(trie_children, vertex_id): - removed = trie_children.get(vertex_id) - return counts[vertex_id] - (removed[0] if removed else 0) - - while True: - root_children = trie_root[2] - live = [] - for index, (value, vertex_id) in enumerate(starts): - if vertex_id is None: - if index not in singletons_drawn: - live.append((index, value, None, 1, 1.0)) - else: - avail = available(root_children, vertex_id) - if avail > 0: - removed = root_children.get(vertex_id) - mass = ((weights[vertex_id] - - (removed[1] if removed else 0.0)) - if weighted else float(avail)) - live.append((index, value, vertex_id, avail, mass)) - if not live: - return - - if weighted: - total_mass = sum(entry[4] for entry in live) - pick = rng.random() * max(total_mass, 0.0) - chosen = live[-1] - for entry in live: - pick -= entry[4] - if pick <= 0: - chosen = entry - break - else: - pick = rng.randrange(sum(entry[3] for entry in live)) - chosen = live[-1] - for entry in live: - pick -= entry[3] - if pick < 0: - chosen = entry - break - index, value, vertex_id = chosen[0], chosen[1], chosen[2] - if vertex_id is None: - singletons_drawn.add(index) - yield (value,) - continue - - path = [vertex_id] - current = vertex_id - trie_cursor = trie_root[2].get(vertex_id) - while True: - children = adjacency.get(current) - if not children: - break - trie_children = trie_cursor[2] if trie_cursor else {} - live_children = [] - for child in children: - avail = available(trie_children, child) - if avail <= 0: - continue - if weighted: - removed = trie_children.get(child) - mass = edges[(current, child)] * ( - weights[child] - - (removed[1] if removed else 0.0)) - live_children.append((child, avail, mass)) - else: - live_children.append((child, avail, float(avail))) - if weighted: - total_mass = sum(mass for _, _, mass in live_children) - pick = rng.random() * max(total_mass, 0.0) - current = live_children[-1][0] - for child, _, mass in live_children: - pick -= mass - if pick <= 0: - current = child - break - else: - pick = rng.randrange( - sum(avail for _, avail, _ in live_children)) - current = live_children[-1][0] - for child, avail, _ in live_children: - pick -= avail - if pick < 0: - current = child - break - path.append(current) - trie_cursor = (trie_cursor[2].get(current) - if trie_cursor else None) - - # Record the drawn path in the trie: below position i the - # removed suffix weight is the product of the remaining edge - # frequencies (1 at the terminal). - suffix_weights = [1.0] * len(path) - for i in range(len(path) - 2, -1, -1): - suffix_weights[i] = (edges[(path[i], path[i + 1])] - * suffix_weights[i + 1]) - cursor = trie_root - for vertex, suffix_weight in zip(path, suffix_weights): - child = cursor[2].get(vertex) - if child is None: - child = [0, 0.0, {}] - cursor[2][vertex] = child - child[0] += 1 - child[1] += suffix_weight - cursor = child - yield tuple(segments[i] for i in path) - - def display(self): - for key, nextkeys in sorted(self.items(), key=lambda x: x): - print('***', key.position, key.value) - for nextkey, frequency in nextkeys.items(): - print(nextkey.value, frequency)