diff --git a/.gitignore b/.gitignore index 734ec00..f6294e7 100644 --- a/.gitignore +++ b/.gitignore @@ -146,4 +146,7 @@ build_documentation .DS_Store # Claude Code -.claude/ \ No newline at end of file +.claude/ + +# Wuggy binary chain caches (written next to language data files) +*.graph.pkl diff --git a/dev-requirements.txt b/dev-requirements.txt index 9b1cba5..afbc116 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,9 +1,8 @@ autopep8==2.0.4 pycodestyle==2.11.0 pylint==3.0.3 -python-Levenshtein==0.25.0 setuptools==69.1.0 wheel==0.42.0 twine==5.0.0 pdoc3==0.10.0 -statsmodels==0.14.1 \ No newline at end of file +statsmodels==0.14.1 diff --git a/documentation/design/extract-graph-kernel.prompt.md b/documentation/design/extract-graph-kernel.prompt.md new file mode 100644 index 0000000..e896dbd --- /dev/null +++ b/documentation/design/extract-graph-kernel.prompt.md @@ -0,0 +1,108 @@ +# Task prompt: sketch the extraction of a domain-agnostic graph kernel + +Paste everything below the line into a new agent session, working from +the repository root on branch `claude/wuggy-2.0-reengineering`. + +--- + +You are working in the Wuggy repository (a multilingual pseudoword +generator) on the local, unpushed branch `claude/wuggy-2.0-reengineering`. +The 2.0 rewrite turned the core bigram store into an interned integer DAG +with NumPy edge arrays, boolean-mask filters, exact path counting, and +uniform/weighted sampling. That core is in +`wuggy/utilities/segmentgraph.py` (class `SegmentGraph`, vertex namedtuple +`Vertex`). Background on why this is worth doing is in +`documentation/design/positional-graph-abstraction.md` — read it first. + +## Goal + +Sketch — and, if it comes out clean, implement — the extraction of a +**domain-agnostic weighted positional-DAG kernel** out from under the +Wuggy-specific `SegmentGraph`, with **no change in observable behavior**. +The deliverable is primarily a design: the module boundary, the kernel's +public interface, and exactly which state and methods move versus stay. +Implementing the refactor is welcome if it is verifiable against the +existing harness, but a crisp proposal with a skeleton is an acceptable +stopping point — do not force a half-finished rewrite. + +## The intended seam + +The kernel (proposed name `PositionalGraph`, in +`wuggy/utilities/positionalgraph.py`) should own everything that has no +knowledge of language: + +- symbol interning and the `(position, symbol)` vertex identity, treating + each symbol payload as an **opaque hashable** — the kernel must not + reference `.letters`, `.hidden`, `Segment`, `SegmentH`, `separator`, + `transform`, or any `language_plugin`; +- the flat edge arrays (`edge_src/dst/freq/pos`), the `edge_mask`, the + position slices, and the view/mask machinery (`_view`, + `_invalidate_caches`, `_edges`, `_adjacency`); +- generic filter *primitives*: an edge-mask filter and a vertex-predicate + / vertex-pass-array filter, plus the numeric `frequency_filter` band + logic (which is already language-free); +- `prune` / `_prune_in_place`, `set_start_vertices`, `get_frequencies`, + `count_paths`, `_completion_counts`, and all of `generate` / + `_generate_exhaustive` / `_generate_sampled` (the three search modes); +- cache serialization of the array state, with the linguistic payload + reconstruction left to the subclass (see `to_cache_state` / + `from_cache_state`, which currently reach into `language_plugin.Segment` + — that reconstruction must move to the Wuggy layer). + +`SegmentGraph` then becomes a thin layer (subclass or composition — you +decide, argue the trade-off) that adds only the linguistic parts: + +- `load()` parsing a plugin data file via `language_plugin.separator` and + `language_plugin.transform`; +- `attribute_filter` and `segmentset_filter`, re-expressed in terms of the + kernel's generic vertex-predicate primitive (`segmentset_filter`'s use + of `segment.letters` is the clearest linguistic dependency; + `attribute_filter`'s `getattr(segment, attribute)` is nearly generic); +- `build_limit_frequencies` (uses named fields); +- the Segment/SegmentH reconstruction half of the cache round-trip. + +Watch for: the mapping-compat methods (`__len__`, `__contains__`, `keys`, +`items`, `display`) are used by cold paths and tests — keep them working, +and decide which layer they belong to. `WuggyGenerator` refers to +`segment_graph`/`segment_graphs` and calls `attribute_filter`, +`frequency_filter`, `prune`, `set_start_vertices`, `generate(mode=...)`, +`get_frequencies`, `count_paths`; that public surface must not change. + +## Hard constraints + +- **No behavioral change.** This is a pure refactor. +- Keep the existing public API of `SegmentGraph` and `WuggyGenerator` + intact (the rename to graph terminology already happened this branch; + do not revert or re-alias it). +- Python only (numpy + rapidfuzz are the runtime deps). No new deps. + +## How to verify + +There is an equivalence harness and a sampling test suite. Set up a venv +with `numpy` and `rapidfuzz`, then: + +1. `python tests/synthetic_plugin/make_data.py` (regenerates committed + synthetic data deterministically; should be a no-op). +2. Capture behavior on the current commit **before** refactoring: + `python tests/capture_behavior.py --out /tmp/before.json`. +3. Do the extraction. +4. `python tests/capture_behavior.py --out /tmp/after.json` and confirm + `diff /tmp/before.json /tmp/after.json` is empty — this proves edge + dumps (type and token weighting), start vertices, exhaustive + generation sets, and all statistics are byte-identical. +5. `python tests/test_sampling.py` must pass (path counts, set-equality + of all three generate modes, seeded determinism, weighted bias, + generator API integration). + +If you implement it, commit on the same branch with a message explaining +the seam; do not push. If you only sketch it, write the proposal to +`documentation/design/graph-kernel-extraction-plan.md` with the concrete +method/state partition and the proposed kernel interface. + +## Deliverable + +Either a verified refactor (both checks green) or a written extraction +plan precise enough to implement without rediscovering the seam — the +exact list of what moves, what stays, the kernel's public signature, and +how `SegmentGraph` re-expresses `attribute_filter` / `segmentset_filter` +on top of it. diff --git a/documentation/design/positional-graph-abstraction.md b/documentation/design/positional-graph-abstraction.md new file mode 100644 index 0000000..496a2cb --- /dev/null +++ b/documentation/design/positional-graph-abstraction.md @@ -0,0 +1,121 @@ +# The positional-graph abstraction + +*Design note, written during the 2.0 reengineering (branch +`claude/wuggy-2.0-reengineering`).* + +## Context + +The 2.0 rewrite replaced Wuggy's dict-of-dicts bigram store with an +interned integer DAG held in flat arrays +([`wuggy/utilities/segmentgraph.py`](../../wuggy/utilities/segmentgraph.py), +class `SegmentGraph`). Filters became boolean masks over the edge array, +`prune()` became a single backward-reachability sweep, and generation +gained exact path counting (`count_paths`) plus uniform and weighted +sampling-without-replacement modes, all driven by one backward dynamic +programming sweep over the layers. + +A consequence worth recording: almost none of that machinery is about +language. It is a general data structure that happens to be carrying +subsyllabic segments today. + +## What the structure actually is + +Strip away the segments and lexicons and it is: + +> **A weighted, layered DAG — equivalently an acyclic weighted +> finite-state automaton — over positional symbols, supporting +> composable hard-constraint masks, exact path counting, and unbiased +> sampling without replacement, all driven by one backward DP.** + +Each *position* is a layer; each vertex is a `(position, symbol)` pair; +edges connect consecutive layers with weights equal to co-occurrence +counts (or summed token frequency). Every attested and every generable +sequence is a path from a start vertex to a vertex in the final layer. + +## The four reusable primitives + +1. **Compact representation of a combinatorial set as paths.** A set of + sequences too large to materialize is held as a small graph. + +2. **Constrain without materializing.** Each filter is a boolean AND + over the edge array; the admissible set is narrowed by intersecting + masks, never by enumerating and rejecting. This is what beats + rejection sampling when the admissible set is a tiny fraction of the + whole. + +3. **Count exactly.** `count_paths` answers a #P-style question that is + tractable *because the graph is a DAG* — one DP sweep. The weighted + version of the same sweep is the **partition function** of the + positional distribution. + +4. **Sample uniformly or by weight, without replacement, exactly.** + +## Applications, sorted by how cleanly they map + +### Maps directly (same machinery, different alphabet) + +- **Template-matched generation in general.** Wuggy is one instance of + "generate items resembling a template, varying a controlled proportion + of elements, drawn from attested statistics." The same engine drives + procedural names, style-matched synthetic/fuzz test corpora, or + de-novo biological sequence candidates (peptides, oligos, SMILES + fragments) under positional-composition constraints and attested + transition frequencies. + +- **Wordlikeness / typicality scoring, not just generation.** Transition + frequencies are already computed as a statistic; exposed as a + normalized scorer (transition-product ÷ partition function) it is a + general "how typical is this string for this lexicon" function — + psycholinguistic norming, narrow-domain spell/OCR/ASR candidate + rescoring, structural anomaly detection (low weight = atypical). + +- **Design-space analytics for experiments.** "How many stimuli survive + these controls?" answered instantly, plus quantities the partition + function unlocks: **entropy of the constrained sub-language** and **KL + divergence between two constrained sub-languages** — e.g. how much a + frequency band or a segment restriction actually shrinks and reshapes + the space. + +### Maps with a modest extension + +- **Layered constraint satisfaction / solution counting.** Read each + position as a CSP variable and the masks as a constraint store: + crossword filling, positionally-constrained scheduling, configurable- + product (feature-model) enumeration. Uniform sampling of *solutions* + and exact solution-counting are cheap here for the same DAG reason. + +## Honest scope boundaries + +- The model is **first-order and acyclic.** Higher-order context means + vertices become `(position, k-gram)` — same machinery, more vertices. + True cycles (a non-positional automaton) break the layered backward + sweep and would need a general topological-order DP, or matrix methods + in the cyclic case. + +- The weighting is a **product of raw frequencies, not a smoothed + probability model.** Fine for ranking and relative comparison, but + unseen transitions are hard zeros; treat it as a partition function + over attested mass, not a calibrated language model. + +- The masks express **hard** constraints well; soft/weighted constraints + beyond the frequency weighting are not modeled. + +## Implication: the extraction seam + +The core already contains no linguistics. This argues for a clean seam +between: + +- a **domain-agnostic kernel** — call it a `PositionalGraph` / + weighted-DAG module — owning interning, edge arrays, masks, counting, + and the three sampling modes; and + +- the **Wuggy-specific layer** — segments, ONC syllabification, + lexicality, OLD20, plugin-driven parsing — sitting on top of it. + +That is also exactly the boundary worth compiling if the counting-and- +sampling kernel ever becomes a Rust / `extendr` core: the kernel is the +part worth reusing and the part worth making fast, and it is separable +from everything that makes Wuggy specifically about pseudowords. + +See [`extract-graph-kernel.prompt.md`](extract-graph-kernel.prompt.md) +for a task prompt that sketches this extraction. diff --git a/requirements.txt b/requirements.txt index b0fb0b3..c2eae79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -Levenshtein >= 0.12.0 +numpy >= 1.22 +rapidfuzz >= 3.0 statsmodels >= 0.12.1 diff --git a/setup.py b/setup.py index 4819809..5d1bdf3 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="wuggy", - version="1.2.0", + version="2.0.0", author="Emmanuel Keuleers", author_email="E.A.Keuleers@tilburguniversity.edu", description="Wuggy: A multilingual pseudoword generator", diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/capture_behavior.py b/tests/capture_behavior.py new file mode 100644 index 0000000..9428e45 --- /dev/null +++ b/tests/capture_behavior.py @@ -0,0 +1,167 @@ +""" +Capture the observable behavior of the Wuggy core as order-independent data. + +Run against two versions of the code and diff the JSON outputs to prove +behavioral equivalence: + + python tests/capture_behavior.py --out baseline.json # on old code + python tests/capture_behavior.py --out new.json # on new code + diff <(jq -S . baseline.json) <(jq -S . new.json) + +Everything recorded is independent of dict ordering and random shuffling: +sorted edge dumps, exhaustive generation sets (hashed), and statistics for +a deterministic sample of candidates. +""" +import argparse +import codecs +import hashlib +import json +import sys +from fractions import Fraction +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO)) +sys.path.insert(1, str(REPO / 'tests')) + +from wuggy import WuggyGenerator # noqa: E402 +from wuggy.utilities.segmentgraph import SegmentGraph # noqa: E402 +from synthetic_plugin.synthetic import SyntheticLanguagePlugin # noqa: E402 + +DATA_DIR = REPO / 'tests' / 'synthetic_plugin' +ENUMERATION_GUARD = 500_000 +STAT_SAMPLE = 150 + + +def jsonable(value): + if isinstance(value, Fraction): + return {'fraction': [value.numerator, value.denominator]} + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + # Normalize: the rewrite stores all frequencies as floats where the + # old code mixed ints and floats. + return round(float(value), 10) + if isinstance(value, dict): + return {str(k): jsonable(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [jsonable(v) for v in value] + return value + + +def canonical(sequence): + """Order-independent canonical string for a generated segment sequence.""" + return '|'.join( + f'{segment.sequence_length}:{segment.segment_length}:{segment.letters}' + for segment in sequence) + + +def edge_dump(graph): + edges = [] + for key, nextkeys in graph.items(): + for nextkey, frequency in nextkeys.items(): + edges.append([key.position, list(key.value), + nextkey.position, list(nextkey.value), + float(frequency)]) + edges.sort() + return edges + + +def start_vertex_dump(graph): + return sorted([key.position, list(key.value)] for key in graph.start_vertices) + + +def enumerate_all(graph): + """Fully drain graph.generate(); returns {canonical: sequence}.""" + sequences = {} + for sequence in graph.generate(): + sequences[canonical(sequence)] = sequence + if len(sequences) > ENUMERATION_GUARD: + raise RuntimeError('enumeration exceeded guard; shrink test data') + return sequences + + +def summarize(sequences): + ordered = sorted(sequences) + return { + 'count': len(ordered), + 'sha256': hashlib.sha256('\n'.join(ordered).encode()).hexdigest(), + 'sample': ordered[:400], + } + + +def capture_statistics(generator, sequences): + """Statistics for a deterministic sample of enumerated sequences.""" + plugin = generator.language_plugin + stats = {} + for canon in sorted(sequences)[:STAT_SAMPLE]: + sequence = sequences[canon] + stats[canon] = { + name: jsonable(getattr(plugin, 'statistic_' + name)(generator, sequence)) + for name in generator.supported_statistics} + return stats + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--out', required=True) + args = parser.parse_args() + + generator = WuggyGenerator() + generator.load('synthetic_test', + local_language_plugin=SyntheticLanguagePlugin()) + result = {'chain_edges_type': edge_dump(generator.segment_graph)} + + token_graph = SegmentGraph(generator.language_plugin) + with codecs.open(DATA_DIR / 'data.txt', 'r', encoding='utf-8') as f: + token_graph.load(f, token=True) + result['chain_edges_token'] = edge_dump(token_graph) + + lookup = sorted(generator.lookup_lexicon.items()) + one_syllable = [w for w, seg in lookup if '-' not in seg][:5] + two_syllable = [w for w, seg in lookup if '-' in seg][:4] + result['reference_words'] = one_syllable + two_syllable + + result['references'] = {} + for word in result['reference_words']: + segments = generator.lookup_reference_segments(word) + generator.set_reference_sequence(segments) + reference = generator.reference_sequence + maxpos = len(reference) - 1 + entry = { + 'segments': segments, + 'ref_frequencies': jsonable(generator.reference_sequence_frequencies), + 'ref_statistics': jsonable(generator.reference_statistics), + } + + plain = generator.segment_graph.prune(maxpos) + plain.set_start_vertices(reference) + entry['startkeys_nofilter'] = start_vertex_dump(plain) + entry['nofilter'] = summarize(enumerate_all(plain)) + + attr_subgraph = generator.segment_graph.attribute_filter( + reference, 'segment_length') + attr_pruned = attr_subgraph.prune(maxpos) + attr_pruned.set_start_vertices(reference) + entry['attr_edges'] = edge_dump(attr_pruned) + attr_sequences = enumerate_all(attr_pruned) + entry['attr'] = summarize(attr_sequences) + entry['attr_stats'] = capture_statistics(generator, attr_sequences) + + freq_subgraph = attr_subgraph.frequency_filter(reference, 4, 4) + freq_pruned = freq_subgraph.prune(maxpos) + freq_pruned.set_start_vertices(reference) + entry['attr_freq4'] = summarize(enumerate_all(freq_pruned)) + + result['references'][word] = entry + + with open(args.out, 'w', encoding='utf-8') as f: + json.dump(result, f, indent=1, sort_keys=True) + counts = [result['references'][w]['nofilter']['count'] + for w in result['reference_words']] + print(f'captured {len(counts)} references, ' + f'enumeration sizes: {counts}', file=sys.stderr) + + +if __name__ == '__main__': + main() diff --git a/tests/synthetic_plugin/__init__.py b/tests/synthetic_plugin/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/synthetic_plugin/data.txt b/tests/synthetic_plugin/data.txt new file mode 100644 index 0000000..655f9d2 --- /dev/null +++ b/tests/synthetic_plugin/data.txt @@ -0,0 +1,220 @@ +baak b:aa:k 4 +bast b:a:st 8 +beelwaast b:ee:l-w:aa:st 4 +beepmiel b:ee:p-m:ie:l 15 +bil b:i:l 4 +book b:oo:k 32 +bund b:u:nd 70 +daagein d:aa:-g:ei:n 4 +dam d:a:m 13 +daziet d:a:-z:ie:t 1 +deis d:ei:s 2 +derman d:e:r-m:a:n 20 +dind d:i:nd 2 +doom d:oo:m 4 +dork d:o:rk 1 +dulstund d:u:l-st:u:nd 11 +dunteel d:u:n-t:ee:l 1 +dup d:u:p 277 +dutwoost d:u:t-w:oo:st 9 +faar f:aa:r 93 +fak f:a:k 6 +fakkrark f:a:k-kr:a:rk 12 +fan f:a:n 8 +fapstel f:a:p-st:e:l 3 +fekroop f:e:-kr:oo:p 2 +fi f:i: 2 +filfur f:i:l-f:u:r 45 +foo f:oo: 1 +foond f:oo:nd 39 +foorkpam f:oo:rk-p:a:m 6 +fu f:u: 1 +fust f:u:st 7 +gaan g:aa:n 2 +geikzeim g:ei:k-z:ei:m 7 +gerkpork g:e:rk-p:o:rk 3 +giengeem g:ie:n-g:ee:m 1 +gikei g:i:-k:ei: 7 +gond g:o:nd 1 +goolzup g:oo:l-z:u:p 9 +goonstiep g:oo:n-st:ie:p 27 +goorknand g:oo:rk-n:a:nd 4 +gund g:u:nd 3 +kaam k:aa:m 42 +kaasnool k:aa:s-n:oo:l 6 +kaastpoom k:aa:st-p:oo:m 6 +kark k:a:rk 2 +kees k:ee:s 26 +keest k:ee:st 1 +keipfust k:ei:p-f:u:st 12 +keist k:ei:st 3 +kel k:e:l 4 +kingul k:i:n-g:u:l 28 +kondreet k:o:nd-r:ee:t 2 +koorplaand k:oo:r-pl:aa:nd 24 +kooskri k:oo:s-kr:i: 1 +krannoos kr:a:n-n:oo:s 2 +kreendkrir kr:ee:nd-kr:i:r 175 +kreest kr:ee:st 12 +kreikraat kr:ei:-kr:aa:t 4 +kreilfit kr:ei:l-f:i:t 18 +kreind kr:ei:nd 9 +krorwo kr:o:r-w:o: 6 +krusfien kr:u:s-f:ie:n 4 +leendstir l:ee:nd-st:i:r 7 +leenmu l:ee:n-m:u: 1 +leerkstin l:ee:rk-st:i:n 14 +leitwoo l:ei:t-w:oo: 109 +lendsnek l:e:nd-sn:e:k 101 +lielres l:ie:l-r:e:s 8 +lirlam l:i:r-l:a:m 11 +listdaal l:i:st-d:aa:l 16 +listkroo l:i:st-kr:oo: 2 +lutweil l:u:t-w:ei:l 1 +maaklein m:aa:k-l:ei:n 46 +maan m:aa:n 1 +maas m:aa:s 1 +meelweist m:ee:l-w:ei:st 25 +meitvep m:ei:t-v:e:p 4 +miem m:ie:m 9 +miest m:ie:st 12 +mind m:i:nd 15 +mitkroork m:i:t-kr:oo:rk 6 +morkron m:o:r-kr:o:n 8 +motbap m:o:t-b:a:p 21 +naatzaark n:aa:t-z:aa:rk 2 +nabaark n:a:-b:aa:rk 16 +neen n:ee:n 47 +neinen n:ei:-n:e:n 4 +niest n:ie:st 27 +nilplees n:i:l-pl:ee:s 1 +nutsiet n:u:t-s:ie:t 5 +panplund p:a:n-pl:u:nd 13 +park p:a:rk 66 +peend p:ee:nd 4 +peer p:ee:r 14 +petkrie p:e:t-kr:ie: 6 +piek p:ie:k 15 +piep p:ie:p 17 +pierk p:ie:rk 281 +pierkreep p:ie:rk-r:ee:p 166 +piespeem p:ie:s-p:ee:m 4 +plak pl:a:k 18 +plam pl:a:m 4 +plas pl:a:s 127 +plast pl:a:st 1 +pleerpoork pl:ee:r-p:oo:rk 23 +pleissnet pl:ei:s-sn:e:t 146 +plook pl:oo:k 9 +plorzat pl:o:r-z:a:t 5 +pond p:o:nd 1 +pooklies p:oo:k-l:ie:s 12 +poon p:oo:n 8 +poorkpot p:oo:rk-p:o:t 209 +popfeest p:o:p-f:ee:st 4 +porkmoot p:o:rk-m:oo:t 1 +pur p:u:r 4 +puttrast p:u:t-tr:a:st 20 +ra r:a: 5 +raam r:aa:m 1 +raanlool r:aa:n-l:oo:l 8 +rap r:a:p 8 +rat r:a:t 2 +reekgust r:ee:k-g:u:st 17 +reep r:ee:p 28 +reinpip r:ei:n-p:i:p 3 +reipkrust r:ei:p-kr:u:st 2 +reksnup r:e:k-sn:u:p 14 +rorlom r:o:r-l:o:m 2 +rund r:u:nd 6 +rurklip r:u:rk-l:i:p 9 +seindkrik s:ei:nd-kr:i:k 9 +seip s:ei:p 9 +seir s:ei:r 19 +senmeit s:e:n-m:ei:t 13 +sie s:ie: 28 +siep s:ie:p 52 +silkeil s:i:l-k:ei:l 1 +sindpleit s:i:nd-pl:ei:t 45 +snaap sn:aa:p 9 +snaatfeel sn:aa:t-f:ee:l 5 +snal sn:a:l 2 +snarzem sn:a:r-z:e:m 4 +sneerk sn:ee:rk 1 +sneik sn:ei:k 2 +sneir sn:ei:r 1 +sneis sn:ei:s 33 +sneissties sn:ei:s-st:ie:s 139 +snendtreest sn:e:nd-tr:ee:st 6 +snolmeis sn:o:l-m:ei:s 11 +snoop sn:oo:p 11 +sootfies s:oo:t-f:ie:s 1 +staand st:aa:nd 4 +staat st:aa:t 17 +steendruk st:ee:nd-r:u:k 5 +stest st:e:st 1 +stet st:e:t 2 +stimsnie st:i:m-sn:ie: 4 +stitman st:i:t-m:a:n 3 +stitsneik st:i:t-sn:ei:k 3 +stoop st:oo:p 4 +sund s:u:nd 9 +sus s:u:s 1 +taampleit t:aa:m-pl:ei:t 33 +tat t:a:t 73 +tee t:ee: 14 +tep t:e:p 59 +tie t:ie: 129 +tiend t:ie:nd 39 +tiermand t:ie:r-m:a:nd 11 +tierzaam t:ie:r-z:aa:m 1 +tinal t:i:-n:a:l 216 +tivark t:i:-v:a:rk 7 +toop t:oo:p 1 +toopkroos t:oo:p-kr:oo:s 77 +traap tr:aa:p 22 +traarplost tr:aa:r-pl:o:st 2 +traastseir tr:aa:st-s:ei:r 11 +traasttook tr:aa:st-t:oo:k 3 +transeim tr:a:n-s:ei:m 21 +trark tr:a:rk 12 +tree tr:ee: 1 +treist tr:ei:st 14 +trer tr:e:r 5 +trerkbeip tr:e:rk-b:ei:p 24 +triem tr:ie:m 10 +trirk tr:i:rk 152 +trootroo tr:oo:-tr:oo: 5 +tropmiend tr:o:p-m:ie:nd 1 +tror tr:o:r 12 +trorke tr:o:r-k:e: 11 +trurvurk tr:u:r-v:u:rk 9 +truttiend tr:u:t-t:ie:nd 2 +tul t:u:l 6 +tundfoond t:u:nd-f:oo:nd 1 +veerk v:ee:rk 46 +velsnak v:e:l-sn:a:k 1 +videis v:i:-d:ei:s 1 +vum v:u:m 1 +vustwurk v:u:st-w:u:rk 4 +vut v:u:t 102 +waa w:aa: 1 +waal w:aa:l 10 +waamkrim w:aa:m-kr:i:m 19 +waamrap w:aa:m-r:a:p 9 +week w:ee:k 30 +wees w:ee:s 1 +wieksir w:ie:k-s:i:r 4 +wogeer w:o:-g:ee:r 32 +wok w:o:k 3 +wunsnis w:u:n-sn:i:s 9 +wurpein w:u:r-p:ei:n 2 +zaarree z:aa:r-r:ee: 20 +zan z:a:n 6 +zeirk z:ei:rk 4 +zesloond z:e:s-l:oo:nd 7 +zetren z:e:t-r:e:n 7 +zielvet z:ie:l-v:e:t 34 +zies z:ie:s 4 +zoodek z:oo:-d:e:k 2 +zund z:u:nd 31 diff --git a/tests/synthetic_plugin/lexicon.txt b/tests/synthetic_plugin/lexicon.txt new file mode 100644 index 0000000..032b1d3 --- /dev/null +++ b/tests/synthetic_plugin/lexicon.txt @@ -0,0 +1,220 @@ +baak 4 +bast 8 +beelwaast 4 +beepmiel 15 +bil 4 +book 32 +bund 70 +daagein 4 +dam 13 +daziet 1 +deis 2 +derman 20 +dind 2 +doom 4 +dork 1 +dulstund 11 +dunteel 1 +dup 277 +dutwoost 9 +faar 93 +fak 6 +fakkrark 12 +fan 8 +fapstel 3 +fekroop 2 +fi 2 +filfur 45 +foo 1 +foond 39 +foorkpam 6 +fu 1 +fust 7 +gaan 2 +geikzeim 7 +gerkpork 3 +giengeem 1 +gikei 7 +gond 1 +goolzup 9 +goonstiep 27 +goorknand 4 +gund 3 +kaam 42 +kaasnool 6 +kaastpoom 6 +kark 2 +kees 26 +keest 1 +keipfust 12 +keist 3 +kel 4 +kingul 28 +kondreet 2 +koorplaand 24 +kooskri 1 +krannoos 2 +kreendkrir 175 +kreest 12 +kreikraat 4 +kreilfit 18 +kreind 9 +krorwo 6 +krusfien 4 +leendstir 7 +leenmu 1 +leerkstin 14 +leitwoo 109 +lendsnek 101 +lielres 8 +lirlam 11 +listdaal 16 +listkroo 2 +lutweil 1 +maaklein 46 +maan 1 +maas 1 +meelweist 25 +meitvep 4 +miem 9 +miest 12 +mind 15 +mitkroork 6 +morkron 8 +motbap 21 +naatzaark 2 +nabaark 16 +neen 47 +neinen 4 +niest 27 +nilplees 1 +nutsiet 5 +panplund 13 +park 66 +peend 4 +peer 14 +petkrie 6 +piek 15 +piep 17 +pierk 281 +pierkreep 166 +piespeem 4 +plak 18 +plam 4 +plas 127 +plast 1 +pleerpoork 23 +pleissnet 146 +plook 9 +plorzat 5 +pond 1 +pooklies 12 +poon 8 +poorkpot 209 +popfeest 4 +porkmoot 1 +pur 4 +puttrast 20 +ra 5 +raam 1 +raanlool 8 +rap 8 +rat 2 +reekgust 17 +reep 28 +reinpip 3 +reipkrust 2 +reksnup 14 +rorlom 2 +rund 6 +rurklip 9 +seindkrik 9 +seip 9 +seir 19 +senmeit 13 +sie 28 +siep 52 +silkeil 1 +sindpleit 45 +snaap 9 +snaatfeel 5 +snal 2 +snarzem 4 +sneerk 1 +sneik 2 +sneir 1 +sneis 33 +sneissties 139 +snendtreest 6 +snolmeis 11 +snoop 11 +sootfies 1 +staand 4 +staat 17 +steendruk 5 +stest 1 +stet 2 +stimsnie 4 +stitman 3 +stitsneik 3 +stoop 4 +sund 9 +sus 1 +taampleit 33 +tat 73 +tee 14 +tep 59 +tie 129 +tiend 39 +tiermand 11 +tierzaam 1 +tinal 216 +tivark 7 +toop 1 +toopkroos 77 +traap 22 +traarplost 2 +traastseir 11 +traasttook 3 +transeim 21 +trark 12 +tree 1 +treist 14 +trer 5 +trerkbeip 24 +triem 10 +trirk 152 +trootroo 5 +tropmiend 1 +tror 12 +trorke 11 +trurvurk 9 +truttiend 2 +tul 6 +tundfoond 1 +veerk 46 +velsnak 1 +videis 1 +vum 1 +vustwurk 4 +vut 102 +waa 1 +waal 10 +waamkrim 19 +waamrap 9 +week 30 +wees 1 +wieksir 4 +wogeer 32 +wok 3 +wunsnis 9 +wurpein 2 +zaarree 20 +zan 6 +zeirk 4 +zesloond 7 +zetren 7 +zielvet 34 +zies 4 +zoodek 2 +zund 31 diff --git a/tests/synthetic_plugin/lookup.txt b/tests/synthetic_plugin/lookup.txt new file mode 100644 index 0000000..d0e1ce6 --- /dev/null +++ b/tests/synthetic_plugin/lookup.txt @@ -0,0 +1,220 @@ +baak b:aa:k +bast b:a:st +beelwaast b:ee:l-w:aa:st +beepmiel b:ee:p-m:ie:l +bil b:i:l +book b:oo:k +bund b:u:nd +daagein d:aa:-g:ei:n +dam d:a:m +daziet d:a:-z:ie:t +deis d:ei:s +derman d:e:r-m:a:n +dind d:i:nd +doom d:oo:m +dork d:o:rk +dulstund d:u:l-st:u:nd +dunteel d:u:n-t:ee:l +dup d:u:p +dutwoost d:u:t-w:oo:st +faar f:aa:r +fak f:a:k +fakkrark f:a:k-kr:a:rk +fan f:a:n +fapstel f:a:p-st:e:l +fekroop f:e:-kr:oo:p +fi f:i: +filfur f:i:l-f:u:r +foo f:oo: +foond f:oo:nd +foorkpam f:oo:rk-p:a:m +fu f:u: +fust f:u:st +gaan g:aa:n +geikzeim g:ei:k-z:ei:m +gerkpork g:e:rk-p:o:rk +giengeem g:ie:n-g:ee:m +gikei g:i:-k:ei: +gond g:o:nd +goolzup g:oo:l-z:u:p +goonstiep g:oo:n-st:ie:p +goorknand g:oo:rk-n:a:nd +gund g:u:nd +kaam k:aa:m +kaasnool k:aa:s-n:oo:l +kaastpoom k:aa:st-p:oo:m +kark k:a:rk +kees k:ee:s +keest k:ee:st +keipfust k:ei:p-f:u:st +keist k:ei:st +kel k:e:l +kingul k:i:n-g:u:l +kondreet k:o:nd-r:ee:t +koorplaand k:oo:r-pl:aa:nd +kooskri k:oo:s-kr:i: +krannoos kr:a:n-n:oo:s +kreendkrir kr:ee:nd-kr:i:r +kreest kr:ee:st +kreikraat kr:ei:-kr:aa:t +kreilfit kr:ei:l-f:i:t +kreind kr:ei:nd +krorwo kr:o:r-w:o: +krusfien kr:u:s-f:ie:n +leendstir l:ee:nd-st:i:r +leenmu l:ee:n-m:u: +leerkstin l:ee:rk-st:i:n +leitwoo l:ei:t-w:oo: +lendsnek l:e:nd-sn:e:k +lielres l:ie:l-r:e:s +lirlam l:i:r-l:a:m +listdaal l:i:st-d:aa:l +listkroo l:i:st-kr:oo: +lutweil l:u:t-w:ei:l +maaklein m:aa:k-l:ei:n +maan m:aa:n +maas m:aa:s +meelweist m:ee:l-w:ei:st +meitvep m:ei:t-v:e:p +miem m:ie:m +miest m:ie:st +mind m:i:nd +mitkroork m:i:t-kr:oo:rk +morkron m:o:r-kr:o:n +motbap m:o:t-b:a:p +naatzaark n:aa:t-z:aa:rk +nabaark n:a:-b:aa:rk +neen n:ee:n +neinen n:ei:-n:e:n +niest n:ie:st +nilplees n:i:l-pl:ee:s +nutsiet n:u:t-s:ie:t +panplund p:a:n-pl:u:nd +park p:a:rk +peend p:ee:nd +peer p:ee:r +petkrie p:e:t-kr:ie: +piek p:ie:k +piep p:ie:p +pierk p:ie:rk +pierkreep p:ie:rk-r:ee:p +piespeem p:ie:s-p:ee:m +plak pl:a:k +plam pl:a:m +plas pl:a:s +plast pl:a:st +pleerpoork pl:ee:r-p:oo:rk +pleissnet pl:ei:s-sn:e:t +plook pl:oo:k +plorzat pl:o:r-z:a:t +pond p:o:nd +pooklies p:oo:k-l:ie:s +poon p:oo:n +poorkpot p:oo:rk-p:o:t +popfeest p:o:p-f:ee:st +porkmoot p:o:rk-m:oo:t +pur p:u:r +puttrast p:u:t-tr:a:st +ra r:a: +raam r:aa:m +raanlool r:aa:n-l:oo:l +rap r:a:p +rat r:a:t +reekgust r:ee:k-g:u:st +reep r:ee:p +reinpip r:ei:n-p:i:p +reipkrust r:ei:p-kr:u:st +reksnup r:e:k-sn:u:p +rorlom r:o:r-l:o:m +rund r:u:nd +rurklip r:u:rk-l:i:p +seindkrik s:ei:nd-kr:i:k +seip s:ei:p +seir s:ei:r +senmeit s:e:n-m:ei:t +sie s:ie: +siep s:ie:p +silkeil s:i:l-k:ei:l +sindpleit s:i:nd-pl:ei:t +snaap sn:aa:p +snaatfeel sn:aa:t-f:ee:l +snal sn:a:l +snarzem sn:a:r-z:e:m +sneerk sn:ee:rk +sneik sn:ei:k +sneir sn:ei:r +sneis sn:ei:s +sneissties sn:ei:s-st:ie:s +snendtreest sn:e:nd-tr:ee:st +snolmeis sn:o:l-m:ei:s +snoop sn:oo:p +sootfies s:oo:t-f:ie:s +staand st:aa:nd +staat st:aa:t +steendruk st:ee:nd-r:u:k +stest st:e:st +stet st:e:t +stimsnie st:i:m-sn:ie: +stitman st:i:t-m:a:n +stitsneik st:i:t-sn:ei:k +stoop st:oo:p +sund s:u:nd +sus s:u:s +taampleit t:aa:m-pl:ei:t +tat t:a:t +tee t:ee: +tep t:e:p +tie t:ie: +tiend t:ie:nd +tiermand t:ie:r-m:a:nd +tierzaam t:ie:r-z:aa:m +tinal t:i:-n:a:l +tivark t:i:-v:a:rk +toop t:oo:p +toopkroos t:oo:p-kr:oo:s +traap tr:aa:p +traarplost tr:aa:r-pl:o:st +traastseir tr:aa:st-s:ei:r +traasttook tr:aa:st-t:oo:k +transeim tr:a:n-s:ei:m +trark tr:a:rk +tree tr:ee: +treist tr:ei:st +trer tr:e:r +trerkbeip tr:e:rk-b:ei:p +triem tr:ie:m +trirk tr:i:rk +trootroo tr:oo:-tr:oo: +tropmiend tr:o:p-m:ie:nd +tror tr:o:r +trorke tr:o:r-k:e: +trurvurk tr:u:r-v:u:rk +truttiend tr:u:t-t:ie:nd +tul t:u:l +tundfoond t:u:nd-f:oo:nd +veerk v:ee:rk +velsnak v:e:l-sn:a:k +videis v:i:-d:ei:s +vum v:u:m +vustwurk v:u:st-w:u:rk +vut v:u:t +waa w:aa: +waal w:aa:l +waamkrim w:aa:m-kr:i:m +waamrap w:aa:m-r:a:p +week w:ee:k +wees w:ee:s +wieksir w:ie:k-s:i:r +wogeer w:o:-g:ee:r +wok w:o:k +wunsnis w:u:n-sn:i:s +wurpein w:u:r-p:ei:n +zaarree z:aa:r-r:ee: +zan z:a:n +zeirk z:ei:rk +zesloond z:e:s-l:oo:nd +zetren z:e:t-r:e:n +zielvet z:ie:l-v:e:t +zies z:ie:s +zoodek z:oo:-d:e:k +zund z:u:nd diff --git a/tests/synthetic_plugin/make_data.py b/tests/synthetic_plugin/make_data.py new file mode 100644 index 0000000..c828099 --- /dev/null +++ b/tests/synthetic_plugin/make_data.py @@ -0,0 +1,58 @@ +""" +Deterministically generate a synthetic language for tests and benchmarks. + +Words are built from CV(C) syllables segmented as onset:nucleus:coda, the +same shape official orthographic plugins use with copy_onc. Running this +script regenerates the committed data files bit-for-bit (fixed seed). +""" +import argparse +import random +from pathlib import Path + +ONSETS = ['b', 'k', 'd', 'f', 'g', 'l', 'm', 'n', 'p', 'r', + 's', 't', 'v', 'w', 'z', 'st', 'tr', 'pl', 'kr', 'sn'] +NUCLEI = ['a', 'e', 'i', 'o', 'u', 'aa', 'ee', 'oo', 'ie', 'ei'] +CODAS = ['', 'k', 'l', 'm', 'n', 'p', 'r', 's', 't', 'st', 'rk', 'nd'] + + +def make_words(n, seed=1234, max_syllables=2): + rng = random.Random(seed) + words = {} + while len(words) < n: + nsyllables = rng.randint(1, max_syllables) + syllables = [(rng.choice(ONSETS), rng.choice(NUCLEI), rng.choice(CODAS)) + for _ in range(nsyllables)] + plain = ''.join(''.join(syllable) for syllable in syllables) + if plain in words: + continue + segmented = '-'.join(':'.join(syllable) for syllable in syllables) + frequency = max(1, int(rng.lognormvariate(2.0, 1.5))) + words[plain] = (segmented, frequency) + return words + + +def write_language(words, outdir): + outdir = Path(outdir) + outdir.mkdir(parents=True, exist_ok=True) + items = sorted(words.items()) + with open(outdir / 'data.txt', 'w', encoding='utf-8') as f: + for plain, (segmented, frequency) in items: + f.write(f'{plain}\t{segmented}\t{frequency}\n') + with open(outdir / 'lookup.txt', 'w', encoding='utf-8') as f: + for plain, (segmented, _) in items: + f.write(f'{plain}\t{segmented}\n') + with open(outdir / 'lexicon.txt', 'w', encoding='utf-8') as f: + for plain, (_, frequency) in items: + f.write(f'{plain}\t{frequency}\n') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--words', type=int, default=220) + parser.add_argument('--seed', type=int, default=1234) + parser.add_argument('--max-syllables', type=int, default=2) + parser.add_argument('--out', default=str(Path(__file__).parent)) + args = parser.parse_args() + write_language( + make_words(args.words, args.seed, args.max_syllables), args.out) + print(f'wrote {args.words} words to {args.out}') diff --git a/tests/synthetic_plugin/synthetic.py b/tests/synthetic_plugin/synthetic.py new file mode 100644 index 0000000..f58fc24 --- /dev/null +++ b/tests/synthetic_plugin/synthetic.py @@ -0,0 +1,12 @@ +from wuggy.plugins.baselanguageplugin import BaseLanguagePlugin + + +class SyntheticLanguagePlugin(BaseLanguagePlugin): + """Minimal plugin over the generated synthetic language data.""" + default_data = 'data.txt' + default_word_lexicon = 'lexicon.txt' + default_neighbor_lexicon = 'lexicon.txt' + default_lookup_lexicon = 'lookup.txt' + + def transform(self, input_sequence, frequency=1): + return self.copy_onc(input_sequence, frequency) diff --git a/tests/test_sampling.py b/tests/test_sampling.py new file mode 100644 index 0000000..984469f --- /dev/null +++ b/tests/test_sampling.py @@ -0,0 +1,166 @@ +""" +Tests for count_paths() and the uniform/weighted generation modes. + +Both sampling modes draw without replacement, so they must yield exactly +the same set of sequences as exhaustive traversal, each exactly once. +Runnable directly: python tests/test_sampling.py +""" +import random +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO)) +sys.path.insert(1, str(REPO / 'tests')) + +from wuggy import WuggyGenerator # noqa: E402 +from synthetic_plugin.synthetic import SyntheticLanguagePlugin # noqa: E402 + +PERMUTATION_SIZE_CAP = 30_000 + + +def make_generator(): + generator = WuggyGenerator() + generator.load('synthetic_test', + local_language_plugin=SyntheticLanguagePlugin()) + return generator + + +def canonical(sequence): + return '|'.join( + f'{s.sequence_length}:{s.segment_length}:{s.letters}' + for s in sequence) + + +def prepared_subgraphs(generator): + """(label, subgraph, reference) for a few filter configurations.""" + lookup = sorted(generator.lookup_lexicon.items()) + references = ([w for w, seg in lookup if '-' not in seg][:3] + + [w for w, seg in lookup if '-' in seg][:2]) + for word in references: + generator.set_reference_sequence( + generator.lookup_reference_segments(word)) + reference = generator.reference_sequence + maxpos = len(reference) - 1 + attr = generator.segment_graph.attribute_filter( + reference, 'segment_length') + pruned = attr.prune(maxpos) + pruned.set_start_vertices(reference) + yield f'{word}/attr', pruned, reference + banded = attr.frequency_filter(reference, 4, 4) + banded = banded.prune(maxpos) + banded.set_start_vertices(reference) + yield f'{word}/attr+freq4', banded, reference + + +def test_permutation_equivalence_and_counts(): + generator = make_generator() + tested = 0 + for label, subgraph, _ in prepared_subgraphs(generator): + expected = subgraph.count_paths() + if expected == 0 or expected > PERMUTATION_SIZE_CAP: + continue + exhaustive = [canonical(s) for s in subgraph.generate()] + assert len(exhaustive) == expected, ( + f'{label}: count_paths {expected} != exhaustive {len(exhaustive)}') + for mode in ('uniform', 'weighted'): + drawn = [canonical(s) for s in + subgraph.generate(mode=mode, rng=random.Random(7))] + assert len(drawn) == expected, ( + f'{label}/{mode}: yielded {len(drawn)}, expected {expected}') + assert set(drawn) == set(exhaustive), ( + f'{label}/{mode}: sequence sets differ from exhaustive') + tested += 1 + assert tested >= 4, f'only {tested} subgraphs were small enough to test' + print(f'permutation equivalence ok over {tested} subgraphs') + + +def test_seeded_determinism(): + generator = make_generator() + for label, subgraph, _ in prepared_subgraphs(generator): + if not 0 < subgraph.count_paths() <= PERMUTATION_SIZE_CAP: + continue + for mode in ('uniform', 'weighted'): + first = [canonical(s) for s in + subgraph.generate(mode=mode, rng=random.Random(123))] + second = [canonical(s) for s in + subgraph.generate(mode=mode, rng=random.Random(123))] + assert first == second, f'{label}/{mode}: seeded runs differ' + break + print('seeded determinism ok') + + +def path_weight(subgraph, sequence, reference): + frequencies = subgraph.get_frequencies(sequence) + weight = 1.0 + for value in frequencies.values(): + weight *= max(value, 1e-12) + return weight + + +def test_weighted_mode_prefers_frequent_paths(): + generator = make_generator() + for label, subgraph, reference in prepared_subgraphs(generator): + if not 1_000 < subgraph.count_paths() <= PERMUTATION_SIZE_CAP: + continue + head = 25 + uniform_weights, weighted_weights = [], [] + for seed in range(20): + drawn = subgraph.generate(mode='uniform', + rng=random.Random(seed)) + uniform_weights.extend( + path_weight(subgraph, s, reference) + for _, s in zip(range(head), drawn)) + drawn = subgraph.generate(mode='weighted', + rng=random.Random(seed)) + weighted_weights.extend( + path_weight(subgraph, s, reference) + for _, s in zip(range(head), drawn)) + mean_uniform = sum(uniform_weights) / len(uniform_weights) + mean_weighted = sum(weighted_weights) / len(weighted_weights) + assert mean_weighted > mean_uniform, ( + f'{label}: weighted mode did not prefer frequent paths ' + f'({mean_weighted:.1f} <= {mean_uniform:.1f})') + print(f'weighted-bias ok on {label}: mean path weight ' + f'{mean_weighted:.1f} (weighted) vs {mean_uniform:.1f} (uniform)') + return + raise AssertionError('no subgraph of suitable size found') + + +def test_generator_api_integration(): + generator = make_generator() + words = sorted(generator.lookup_lexicon)[:3] + for search_mode in ('uniform', 'weighted'): + random.seed(11) + matches = generator.generate_classic( + words, ncandidates_per_sequence=5, max_search_time_per_sequence=3, + search_mode=search_mode) + assert matches, f'no classic matches in {search_mode} mode' + assert all(m['statistics']['lexicality'] == 'N' for m in matches) + # generate_classic leaves its last concentric frequency filter set; + # clear it so the advanced generator starts from the full graph. + generator.clear_frequency_filter() + generator.clear_attribute_filters() + generator.set_reference_sequence( + generator.lookup_reference_segments(words[0])) + generator.set_attribute_filter('segment_length') + generator.set_output_mode('plain') + random.seed(11) + advanced = list(generator.generate_advanced(mode='uniform')) + assert advanced and len(advanced) == len(set(advanced)) + try: + generator.segment_graph.generate(mode='nonsense') + except ValueError: + pass + else: + raise AssertionError('unknown mode did not raise ValueError') + print(f'generator API integration ok ' + f'({len(advanced)} advanced uniform outputs)') + + +if __name__ == '__main__': + test_permutation_equivalence_and_counts() + test_seeded_determinism() + test_weighted_mode_prefers_frequent_paths() + test_generator_api_integration() + print('all sampling tests passed') diff --git a/wuggy/evaluators/ld1nn.py b/wuggy/evaluators/ld1nn.py index 6974d48..bec3978 100644 --- a/wuggy/evaluators/ld1nn.py +++ b/wuggy/evaluators/ld1nn.py @@ -1,6 +1,6 @@ from math import exp -from Levenshtein import distance +from rapidfuzz.distance.Levenshtein import distance def ld1nn(word_sample: [str], diff --git a/wuggy/generators/wuggygenerator.py b/wuggy/generators/wuggygenerator.py index ab4ddd7..78972a1 100644 --- a/wuggy/generators/wuggygenerator.py +++ b/wuggy/generators/wuggygenerator.py @@ -1,9 +1,9 @@ import codecs -import copy import importlib import importlib.util import inspect import os +import pickle import sys from collections import defaultdict, namedtuple from csv import writer @@ -52,7 +52,18 @@ def _language_plugins_base_dir() -> Path: ) from ..plugins.baselanguageplugin import BaseLanguagePlugin -from ..utilities.bigramchain import BigramChain +from ..utilities.segmentgraph import SegmentGraph + + +def _copy_statistics(statistics: Dict) -> Dict: + """ + Snapshot a statistics dict for inclusion in a returned match. Values + are scalars or per-call-created containers, so a one-level copy is + enough to decouple the match from the generator's mutable state. + """ + return {key: (dict(value) if isinstance(value, dict) + else list(value) if isinstance(value, list) else value) + for key, value in statistics.items()} def _loaded_language_plugin_required(func): @@ -85,8 +96,8 @@ def wrapper(*args, **kwargs): class WuggyGenerator(): def __init__(self): - self.bigramchain = None - self.bigramchains = {} + self.segment_graph = None + self.segment_graphs = {} self.supported_official_language_plugin_names = [ "orthographic_basque", "orthographic_dutch", @@ -105,8 +116,8 @@ def __init__(self): "phonetic_french", "phonetic_italian"] self.__official_language_plugin_repository_url = "https://raw.githubusercontent.com/WuggyCode/wuggy_language_plugin_data/master" - self.attribute_subchain = None - self.frequency_subchain = None + self.attribute_subgraph = None + self.frequency_subgraph = None self.reference_sequence = None self.frequency_filter = None self.current_sequence = None @@ -116,11 +127,11 @@ def __init__(self): self.attribute_filters = {} self.default_attributes = [] self.statistics = {} - self.word_lexicon = defaultdict(list) + self.word_lexicon = defaultdict(set) self.neighbor_lexicon = [] self.reference_statistics = {} self.stat_cache = {} - self.sequence_cache = [] + self.sequence_cache = set() self.difference_statistics = {} self.match_statistics = {} self.lookup_lexicon = {} @@ -128,7 +139,7 @@ def __init__(self): def load(self, language_plugin_name: str, local_language_plugin: BaseLanguagePlugin = None) -> None: """ - Loads in a language plugin, if available, and stores the corresponding bigramchains. + Loads in a language plugin, if available, and stores the corresponding segment_graphs. Parameters: language_plugin_name: must be the exact string of an official language plugin (see self.supported_official_language_plugin_names). If you are loading in a local plugin, the name can be anything as long as it does not conflict with an already loaded plugin name. @@ -176,17 +187,44 @@ def load(self, language_plugin_name: str, f".plugins.language_data.{language_plugin_name}.{language_plugin_name}", "wuggy").OfficialLanguagePlugin() - if language_plugin_name not in self.bigramchains: + if language_plugin_name not in self.segment_graphs: default_data_path = os.path.join( self.language_plugin_data_path, language_plugin.default_data) - - data_file = codecs.open(default_data_path, 'r', encoding='utf-8') - self.bigramchains[self.language_plugin_name] = BigramChain( - language_plugin) - self.bigramchains[self.language_plugin_name].load( - data_file) + self.segment_graphs[self.language_plugin_name] = ( + self.__load_segment_graph(language_plugin, default_data_path)) self.__activate(self.language_plugin_name) + @staticmethod + def __load_segment_graph(language_plugin, data_path: str) -> SegmentGraph: + """ + Load a segment graph from its data file, going through a binary + cache next to the data file so the text is parsed only once per + data file version. Cache failures fall back to parsing silently. + """ + cache_path = data_path + '.graph.pkl' + stat = os.stat(data_path) + source_signature = (stat.st_mtime_ns, stat.st_size) + try: + with open(cache_path, 'rb') as f: + cached = pickle.load(f) + if cached.get('source') == source_signature: + return SegmentGraph.from_cache_state( + language_plugin, cached['state']) + except (OSError, ValueError, KeyError, pickle.PickleError, + EOFError, AttributeError): + pass + segment_graph = SegmentGraph(language_plugin) + with open(data_path, 'r', encoding='utf-8') as data_file: + segment_graph.load(data_file) + try: + with open(cache_path, 'wb') as f: + pickle.dump({'source': source_signature, + 'state': segment_graph.to_cache_state()}, + f, protocol=pickle.HIGHEST_PROTOCOL) + except OSError: + pass + return segment_graph + @staticmethod def remove_downloaded_language_plugins() -> None: """ @@ -259,14 +297,14 @@ def download_language_plugin( def __activate(self, name: str) -> None: """ - Activate a language plugin by setting the corresponding bigramchains and lexicon properties. + Activate a language plugin by setting the corresponding segment_graphs and lexicon properties. This deactivates and garbage collects any previously activated language plugin. Should only be called internally, do not call on your own. """ if isinstance(name, type(codecs)): name = name.__name__ - self.bigramchain = self.bigramchains[name] - self.language_plugin = self.bigramchain.language_plugin + self.segment_graph = self.segment_graphs[name] + self.language_plugin = self.segment_graph.language_plugin self.__load_neighbor_lexicon() self.__load_word_lexicon() self.__load_lookup_lexicon() @@ -281,18 +319,16 @@ def __load_word_lexicon(self) -> None: This is currently used internally by __activate only, do not call on your own. """ cutoff = 0 - data_file = codecs.open( - "%s/%s" % (self.language_plugin_data_path, self.language_plugin.default_word_lexicon), - 'r', encoding="utf-8") - self.word_lexicon = defaultdict(list) - lines = data_file.readlines() - for line in lines: - fields = line.strip().split('\t') - word = fields[0] - frequency_per_million = fields[-1] - if float(frequency_per_million) >= cutoff: - self.word_lexicon[word[0], len(word)].append(word) - data_file.close() + self.word_lexicon = defaultdict(set) + with open("%s/%s" % (self.language_plugin_data_path, + self.language_plugin.default_word_lexicon), + 'r', encoding="utf-8") as data_file: + for line in data_file: + fields = line.strip().split('\t') + word = fields[0] + frequency_per_million = fields[-1] + if float(frequency_per_million) >= cutoff: + self.word_lexicon[word[0], len(word)].add(word) def __load_neighbor_lexicon(self) -> None: """ @@ -300,21 +336,16 @@ def __load_neighbor_lexicon(self) -> None: This is currently used internally by __activate only, do not call on your own. """ cutoff = 0 - data_file = codecs.open( - "%s/%s" % - (self.language_plugin_data_path, - self.language_plugin.default_neighbor_lexicon), - 'r', - encoding="utf-8") self.neighbor_lexicon = [] - lines = data_file.readlines() - for line in lines: - fields = line.strip().split('\t') - word = fields[0] - frequency_per_million = fields[-1] - if float(frequency_per_million) >= cutoff: - self.neighbor_lexicon.append(word) - data_file.close() + with open("%s/%s" % (self.language_plugin_data_path, + self.language_plugin.default_neighbor_lexicon), + 'r', encoding="utf-8") as data_file: + for line in data_file: + fields = line.strip().split('\t') + word = fields[0] + frequency_per_million = fields[-1] + if float(frequency_per_million) >= cutoff: + self.neighbor_lexicon.append(word) def __load_lookup_lexicon(self, data_file: bool = None) -> None: """ @@ -323,15 +354,15 @@ def __load_lookup_lexicon(self, data_file: bool = None) -> None: """ self.lookup_lexicon = {} if data_file is None: - data_file = codecs.open( + data_file = open( "%s/%s" % (self.language_plugin_data_path, self.language_plugin.default_lookup_lexicon), 'r', encoding="utf-8") - lines = data_file.readlines() - for line in lines: - fields = line.strip().split(self.language_plugin.separator) + separator = self.language_plugin.separator + for line in data_file: + fields = line.strip().split(separator) reference, representation = fields[0:2] self.lookup_lexicon[reference] = representation data_file.close() @@ -366,11 +397,11 @@ def set_reference_sequence(self, sequence: str) -> None: """ self.reference_sequence = self.language_plugin.transform( sequence).representation - self.reference_sequence_frequencies = self.bigramchain.get_frequencies( + self.reference_sequence_frequencies = self.segment_graph.get_frequencies( self.reference_sequence) self.__clear_stat_cache() for name in self.__get_statistics(): - function = eval("self.language_plugin.statistic_%s" % (name)) + function = getattr(self.language_plugin, 'statistic_' + name) self.reference_statistics[name] = function( self, self.reference_sequence) @@ -415,7 +446,7 @@ def apply_statistics(self, sequence: str = None) -> None: if sequence is None: sequence = self.current_sequence for name in self.statistics: - function = eval("self.language_plugin.statistic_%s" % (name)) + function = getattr(self.language_plugin, 'statistic_' + name) if (sequence, name) in self.stat_cache: self.statistics[name] = self.stat_cache[(sequence, name)] else: @@ -444,7 +475,7 @@ def __clear_sequence_cache(self) -> None: """ Clears the sequence cache. Only used by Wuggy internally. """ - self.sequence_cache = [] + self.sequence_cache = set() def list_output_modes(self) -> [str]: """ @@ -460,7 +491,7 @@ def set_output_mode(self, name: str) -> None: """ if name not in self.list_output_modes(): raise ValueError(f"Output mode {name} is not supported.") - self.output_mode = eval("self.language_plugin.output_%s" % (name)) + self.output_mode = getattr(self.language_plugin, 'output_' + name) def set_attribute_filter(self, name: str) -> None: """ @@ -471,7 +502,7 @@ def set_attribute_filter(self, name: str) -> None: raise ValueError( f"Attribute filter {name} is not supported.") self.attribute_filters[name] = reference_sequence - self.attribute_subchain = None + self.attribute_subgraph = None def set_attribute_filters(self, names: [str]) -> None: """ @@ -486,8 +517,8 @@ def __apply_attribute_filters(self) -> None: This is currently used by Wuggy internally, do not call on your own. """ for attribute, reference_sequence in self.attribute_filters.items(): - subchain = self.attribute_subchain if self.attribute_subchain is not None else self.bigramchain - self.attribute_subchain = subchain.attribute_filter( + subgraph = self.attribute_subgraph if self.attribute_subgraph is not None else self.segment_graph + self.attribute_subgraph = subgraph.attribute_filter( reference_sequence, attribute) def clear_attribute_filters(self) -> None: @@ -508,7 +539,7 @@ def clear_frequency_filter(self) -> None: Clear the previously set frequency filter. """ self.frequency_filter = None - self.frequency_subchain = None + self.frequency_subgraph = None def apply_frequency_filter(self) -> None: """ @@ -517,8 +548,8 @@ def apply_frequency_filter(self) -> None: if self.frequency_filter is None: raise Exception("No frequency filter was set") reference_sequence, lower, upper = self.frequency_filter - subchain = self.attribute_subchain if self.attribute_subchain is not None else self.bigramchain - self.frequency_subchain = subchain.frequency_filter( + subgraph = self.attribute_subgraph if self.attribute_subgraph is not None else self.segment_graph + self.frequency_subgraph = subgraph.frequency_filter( reference_sequence, lower, upper) @_loaded_language_plugin_required @@ -527,7 +558,8 @@ def generate_classic( ncandidates_per_sequence: int = 10, max_search_time_per_sequence: int = 10, subsyllabic_segment_overlap_ratio: Union[Fraction, None] = Fraction(2, 3), match_subsyllabic_segment_length: bool = True, match_letter_length: bool = True, - output_mode: str = "plain", concentric_search: bool = True) -> [Dict]: + output_mode: str = "plain", concentric_search: bool = True, + search_mode: str = "exhaustive") -> [Dict]: """ This is the classic method to generate pseudowords using Wuggy and can be called immediately after loading a language plugin. The defaults for this method are similar to those set in the legacy version of Wuggy, resulting in sensible pseudowords. @@ -548,7 +580,9 @@ def generate_classic( output_mode: output mode for pseudowords, constricted by the output modes supported by the currently loaded language plugin. - concentric_search: enable/disable concentric search. Wuggy operates best and fastest when concentric search is enabled. First, the algorithm will try to generate candidates that exactly match the transition frequencies of the reference word. Then the maximal allowed deviation in transition frequencies will increase by powers of 2 (i.e., +/-2, +/-4, +/-8, etc.). + concentric_search: enable/disable concentric search. Wuggy operates best and fastest when concentric search is enabled. The maximal allowed deviation in transition frequencies starts at +/-2 and increases by powers of 2 (i.e., +/-2, +/-4, +/-8, etc.), so the earliest candidates are those whose transition frequencies track the reference word most closely. + + search_mode: order in which candidates are drawn from the search space. "exhaustive" (default) is the classic shuffled depth-first traversal. "uniform" draws candidates in unbiased random order (uniform sampling without replacement). "weighted" draws candidates with probability proportional to the product of their transition frequencies, so the most word-like candidates tend to be found first. All modes visit each candidate at most once. .. include:: ../../documentation/wuggygenerator/generate_classic.md """ pseudoword_matches = [] @@ -560,14 +594,15 @@ def generate_classic( max_search_time_per_sequence, subsyllabic_segment_overlap_ratio, match_subsyllabic_segment_length, - match_letter_length, output_mode, concentric_search)) + match_letter_length, output_mode, concentric_search, + search_mode)) return pseudoword_matches def __generate_classic_inner( self, input_sequence: str, ncandidates_per_sequence: int, max_search_time: int, subsyllabic_segment_overlap_ratio: Union[Fraction, None], match_subsyllabic_segment_length: bool, match_letter_length: bool, output_mode: str, - concentric_search: bool = True): + concentric_search: bool = True, search_mode: str = "exhaustive"): """ Inner method for generate_classic(), which outputs a list of pseudoword matches for an input sequence. Should only be used by WuggyGenerator internally. @@ -581,24 +616,24 @@ def __generate_classic_inner( f"Sequence {input_sequence} was not found in lexicon {self.current_language_plugin_name}") self.set_reference_sequence(input_sequence_segments) self.set_output_mode(output_mode) - subchain = self.bigramchain + subgraph = self.segment_graph starttime = time() pseudoword_matches = [] frequency_exponent = 1 if match_subsyllabic_segment_length: self.set_attribute_filter("segment_length") self.__apply_attribute_filters() - subchain = self.attribute_subchain + subgraph = self.attribute_subgraph while True: if concentric_search: self.set_frequency_filter( 2**frequency_exponent, 2**frequency_exponent) frequency_exponent += 1 self.apply_frequency_filter() - subchain = self.frequency_subchain - subchain = subchain.clean(len(self.reference_sequence) - 1) - subchain.set_startkeys(self.reference_sequence) - for sequence in subchain.generate(): + subgraph = self.frequency_subgraph + subgraph = subgraph.prune(len(self.reference_sequence) - 1) + subgraph.set_start_vertices(self.reference_sequence) + for sequence in subgraph.generate(mode=search_mode): # Mandatory statistics before finding a suitable match self.clear_statistics() self.set_statistics(["overlap_ratio", "plain_length", "lexicality"]) @@ -618,15 +653,15 @@ def __generate_classic_inner( # (Re)apply all statistics only if match is found: else search becomes unnecessarily slow self.set_all_statistics() self.apply_statistics() - self.sequence_cache.append( + self.sequence_cache.add( self.language_plugin.output_plain(sequence)) match = {"word": input_sequence, "segments": input_sequence_segments, - "pseudoword": self.output_mode(sequence)} - match.update({"statistics": self.statistics, - "difference_statistics": self.difference_statistics}) - - pseudoword_matches.append(copy.deepcopy(match)) + "pseudoword": self.output_mode(sequence), + "statistics": _copy_statistics(self.statistics), + "difference_statistics": _copy_statistics( + self.difference_statistics)} + pseudoword_matches.append(match) if len(pseudoword_matches) >= ncandidates_per_sequence: return pseudoword_matches @@ -637,7 +672,8 @@ def generate_gui( subsyllabic_segment_overlap_ratio: Union[Fraction, None] = Fraction(2, 3), match_subsyllabic_segment_length: bool = True, match_letter_length: bool = True, output_mode: str = "plain", concentric_search: bool = True, - output_type: str = "pseudowords") -> [Dict]: + output_type: str = "pseudowords", + search_mode: str = "exhaustive") -> [Dict]: """ Variant of generate_classic tailored for GUI use. Identical to generate_classic except for the output_type parameter, which controls @@ -658,14 +694,15 @@ def generate_gui( subsyllabic_segment_overlap_ratio, match_subsyllabic_segment_length, match_letter_length, output_mode, concentric_search, - output_type)) + output_type, search_mode)) return pseudoword_matches def __generate_gui_inner( self, input_sequence: str, ncandidates_per_sequence: int, max_search_time: int, subsyllabic_segment_overlap_ratio: Union[Fraction, None], match_subsyllabic_segment_length: bool, match_letter_length: bool, output_mode: str, - concentric_search: bool, output_type: str): + concentric_search: bool, output_type: str, + search_mode: str = "exhaustive"): self.__clear_sequence_cache() self.clear_attribute_filters() self.clear_frequency_filter() @@ -675,24 +712,24 @@ def __generate_gui_inner( f"Sequence {input_sequence} was not found in lexicon {self.current_language_plugin_name}") self.set_reference_sequence(input_sequence_segments) self.set_output_mode(output_mode) - subchain = self.bigramchain + subgraph = self.segment_graph starttime = time() pseudoword_matches = [] frequency_exponent = 1 if match_subsyllabic_segment_length: self.set_attribute_filter("segment_length") self.__apply_attribute_filters() - subchain = self.attribute_subchain + subgraph = self.attribute_subgraph while True: if concentric_search: self.set_frequency_filter( 2**frequency_exponent, 2**frequency_exponent) frequency_exponent += 1 self.apply_frequency_filter() - subchain = self.frequency_subchain - subchain = subchain.clean(len(self.reference_sequence) - 1) - subchain.set_startkeys(self.reference_sequence) - for sequence in subchain.generate(): + subgraph = self.frequency_subgraph + subgraph = subgraph.prune(len(self.reference_sequence) - 1) + subgraph.set_start_vertices(self.reference_sequence) + for sequence in subgraph.generate(mode=search_mode): self.clear_statistics() self.set_statistics(["overlap_ratio", "plain_length", "lexicality"]) if (time() - starttime) >= max_search_time: @@ -713,52 +750,56 @@ def __generate_gui_inner( continue self.set_all_statistics() self.apply_statistics() - self.sequence_cache.append( + self.sequence_cache.add( self.language_plugin.output_plain(sequence)) match = {"word": input_sequence, "segments": input_sequence_segments, - "pseudoword": self.output_mode(sequence)} - match.update({"statistics": self.statistics, - "difference_statistics": self.difference_statistics}) - pseudoword_matches.append(copy.deepcopy(match)) + "pseudoword": self.output_mode(sequence), + "statistics": _copy_statistics(self.statistics), + "difference_statistics": _copy_statistics( + self.difference_statistics)} + pseudoword_matches.append(match) if len(pseudoword_matches) >= ncandidates_per_sequence: return pseudoword_matches @_loaded_language_plugin_required_generator - def generate_advanced(self, clear_cache: bool = True) -> Union[Generator[str, None, None], - Generator[tuple, None, None]]: + def generate_advanced(self, clear_cache: bool = True, + mode: str = "exhaustive") -> Union[Generator[str, None, None], + Generator[tuple, None, None]]: """ Creates a custom generator which can be iterated to return generated pseudowords. The generator's settings, such as output statistics, should be set by you before calling this method. If attributes such as \"output_mode\" are not set, sensible defaults are used. Note that this method is for advanced users and may result in unexpected results if handled incorrectly. + Parameters: + mode: order in which sequences are drawn from the search space. "exhaustive" (default) is the classic shuffled depth-first traversal. "uniform" yields every sequence exactly once in unbiased random order. "weighted" yields every sequence exactly once, sampled without replacement proportionally to the product of its transition frequencies, so the most word-like sequences tend to arrive first. .. include:: ../../documentation/wuggygenerator/generate_advanced.md """ if clear_cache: self.__clear_sequence_cache() if self.output_mode is None: self.set_output_mode("plain") - if len(self.attribute_filters) == 0 and self.frequency_subchain is None: - subchain = self.bigramchain + if len(self.attribute_filters) == 0 and self.frequency_subgraph is None: + subgraph = self.segment_graph if len(self.attribute_filters) != 0: - if self.attribute_subchain is None: + if self.attribute_subgraph is None: self.__apply_attribute_filters() - subchain = self.attribute_subchain + subgraph = self.attribute_subgraph if self.frequency_filter is not None: self.apply_frequency_filter() - subchain = self.frequency_subchain + subgraph = self.frequency_subgraph if self.reference_sequence is not None: - subchain = subchain.clean(len(self.reference_sequence) - 1) - subchain.set_startkeys(self.reference_sequence) + subgraph = subgraph.prune(len(self.reference_sequence) - 1) + subgraph.set_start_vertices(self.reference_sequence) else: warn( "No reference sequence was set. Ignore this message if this was intentional.") - subchain.set_startkeys() - for sequence in subchain.generate(): + subgraph.set_start_vertices() + for sequence in subgraph.generate(mode=mode): if self.language_plugin.output_plain(sequence) in self.sequence_cache: pass else: - self.sequence_cache.append( + self.sequence_cache.add( self.language_plugin.output_plain(sequence)) self.current_sequence = sequence self.apply_statistics() diff --git a/wuggy/plugins/baselanguageplugin.py b/wuggy/plugins/baselanguageplugin.py index b8df8de..94eedec 100644 --- a/wuggy/plugins/baselanguageplugin.py +++ b/wuggy/plugins/baselanguageplugin.py @@ -2,8 +2,9 @@ from collections import namedtuple from fractions import Fraction -# Pylint may report no-member error due to C extension -import Levenshtein +import numpy as np +from rapidfuzz import process +from rapidfuzz.distance import Levenshtein def compute_difference(gen_stat, ref_stat): @@ -134,9 +135,11 @@ def _distance(self, source, target): return Levenshtein.distance(source, target) def _old(self, source, lexicon, n): - distances = (distance for neighbor, - distance in self._neighbors(source, lexicon, n)) - return sum(distances) / float(n) + distances = process.cdist( + [source], lexicon, scorer=Levenshtein.distance, workers=-1)[0] + if len(distances) > n: + distances = np.partition(distances, n - 1)[:n] + return float(distances.sum(dtype=np.int64)) / float(n) def _neighbors(self, source, lexicon, n): neighbors = [] @@ -146,13 +149,15 @@ def _neighbors(self, source, lexicon, n): return neighbors[0:n] def _neighbors_at_distance(self, source, lexicon, distance): - neighbors = [] - for target in lexicon: - if abs(len(target) - len(source)) > distance: - pass - elif Levenshtein.distance(source, target) == 1: - neighbors.append(target) - return neighbors + length = len(source) + candidates = [target for target in lexicon + if abs(len(target) - length) <= distance] + if not candidates: + return [] + distances = process.cdist( + [source], candidates, scorer=Levenshtein.distance, + score_cutoff=1, workers=-1)[0] + return [target for target, d in zip(candidates, distances) if d == 1] @match @difference @@ -168,7 +173,7 @@ def statistic_ned1(self, generator, generated_sequence): @difference def statistic_transition_frequencies(self, generator, generated_sequence): - return generator.bigramchain.get_frequencies(generated_sequence) + return generator.segment_graph.get_frequencies(generated_sequence) def onsetnucleuscoda(self, orthographic_syllable, lang=None): self.oncpattern = lang.oncpattern diff --git a/wuggy/utilities/bigramchain.py b/wuggy/utilities/bigramchain.py deleted file mode 100644 index 1549c02..0000000 --- a/wuggy/utilities/bigramchain.py +++ /dev/null @@ -1,174 +0,0 @@ -import random -from collections import defaultdict, namedtuple - -Link = namedtuple('Link', ['position', 'value']) - - -class BigramChain(defaultdict): - """ - A dictionary storing the next possible value, given a list of input sequences. - """ - - def __init__(self, language_plugin, data=None, encoding='utf-8', size=100, cutoff=1, token=False): - defaultdict.__init__(self, dict) - self.language_plugin = language_plugin - try: - self.hidden_sequence = self.language_plugin.hidden_sequence - except AttributeError: - self.hidden_sequence = False - if data != None: - self.load(data, size=size, cutoff=cutoff, token=token) - self.startkeys = [] - self.limit_frequencies = {} - - def load(self, datafile, size=100, cutoff=1, token=False): - lines = datafile.readlines() - for i, line in enumerate(lines): - fields = line.strip('\n\t').split(self.language_plugin.separator) - reference, input_sequence, frequency = fields - frequency = float(frequency) if token == True else 1 - frequency = 1 - sequence = (self.language_plugin.transform( - input_sequence, frequency)) - n = len(sequence.representation) - if frequency >= cutoff and random.randint(1, 100) <= size: - for j in range(n): - key = Link(j, sequence.representation[j]) - if j+1 < n: - next_key = Link(j+1, sequence.representation[j+1]) - self[key][next_key] = self[key].get( - next_key, 0)+sequence.frequency - else: - pass - datafile.close() - self.set_startkeys() - - def set_startkeys(self, reference_sequence=None, fields=None): - if fields == None: - fields = self.language_plugin.default_fields - if reference_sequence == None: - self.startkeys = dict([(key, 0) - for key in self.keys() if key.position == 0]) - else: - self.startkeys = {} - for key in self.keys(): - if key.position == 0: - self.startkeys[key] = 0 - - def get_frequencies(self, reference_sequence): - frequencies = {} - for position in range(len(reference_sequence)-1): - key = Link(position, reference_sequence[position]) - nextkey = Link(position+1, reference_sequence[position+1]) - try: - frequency = self[key][nextkey] - except KeyError: - frequency = 0 - frequencies[position] = frequency - return frequencies - - def build_limit_frequencies(self, fields): - limits = defaultdict(dict) - for key, nextkeys in self.items(): - position, value = key - subkey_a = (position, tuple( - [value.__getattribute__(field) for field in fields])) - for nextkey, frequency in nextkeys.items(): - position, value = nextkey - subkey_b = (position, tuple( - [value.__getattribute__(field) for field in fields])) - subkey = (subkey_a, subkey_b) - minfrequency = limits[subkey].get('min', frequency) - limits[subkey]['min'] = min(minfrequency, frequency) - maxfrequency = limits[subkey].get('max', frequency) - limits[subkey]['max'] = max(maxfrequency, frequency) - self.limit_frequencies[tuple(fields)] = limits - - def frequency_filter(self, reference_sequence, lower, upper, kind='dev'): - result = BigramChain(self.language_plugin) - frequencies = self.get_frequencies(reference_sequence) - for key, nextkeys in self.items(): - try: - if kind == 'dev': - minfreq = frequencies[key.position]-lower - maxfreq = frequencies[key.position]+upper - elif kind == 'limit': - minfreq = lower - maxfreq = upper - except: - pass - else: - for nextkey, frequency in nextkeys.items(): - if minfreq <= frequency <= maxfreq: - result[key][nextkey] = frequency - result = result.clean(len(reference_sequence)-1) - result.set_startkeys() - return result - - def segmentset_filter(self, reference_sequence, segmentset): - segmentset = segmentset.union(set(('^', '$'))) - result = BigramChain(self.language_plugin) - for key, nextkeys in self.items(): - if key.value.letters in segmentset: - for nextkey, frequency in nextkeys.items(): - if nextkey.value.letters in segmentset: - result[key][nextkey] = frequency - result = result.clean(len(reference_sequence)-1) - result.set_startkeys() - return result - - def attribute_filter(self, reference_sequence, attribute): - result = BigramChain(self.language_plugin) - if type(reference_sequence[0]) == self.language_plugin.Segment: - for key, nextkeys in self.items(): - try: - if key.value.__getattribute__(attribute) == reference_sequence[ - key.position].__getattribute__(attribute): - result[key] = self[key] - except IndexError: - pass - else: - for key, nextkeys in self.items(): - try: - if key.value.__getattribute__(attribute) == reference_sequence[key.position]: - result[key] = self[key] - except IndexError: - pass - return result - - def clean(self, maxpos): - """ - Remove chains that can not be completed. - """ - result = BigramChain(self.language_plugin) - for key, nextkeys in self.items(): - for nextkey, frequency in nextkeys.items(): - if nextkey in self or nextkey.position == maxpos: - result[key][nextkey] = frequency - if len(self) == len(result): - return result - else: - return result.clean(maxpos) - - def generate(self, startkeys=None): - if startkeys is None: - startkeys = self.startkeys - startkeys = list(startkeys.items()) - random.shuffle(startkeys) - startkeys = dict(startkeys) - if len(self) > 0: - for key in startkeys: - if key not in self: - yield (key.value,) - else: - next_keys = self[key] - for result in self.generate(next_keys): - yield (key.value,)+result - else: - raise Exception('LinkError') - - def display(self): - for key, nextkeys in sorted(self.items(), key=lambda x: x): - print('***', key.position, key.value) - for nextkey, frequency in nextkeys.items(): - print(nextkey.value, frequency) diff --git a/wuggy/utilities/positionalgraph.py b/wuggy/utilities/positionalgraph.py new file mode 100644 index 0000000..8b49f34 --- /dev/null +++ b/wuggy/utilities/positionalgraph.py @@ -0,0 +1,576 @@ +import random +from collections import defaultdict, namedtuple + +import numpy as np + +Vertex = namedtuple('Vertex', ['position', 'value']) + +_SENTINEL = object() + + +class PositionalGraph: + """ + A weighted, layered DAG over (position, symbol) vertices storing the next + possible values, given a list of input sequences. + + Symbols are opaque hashable payloads; this class knows nothing about what + they mean. Vertices are interned to integer ids; edges live in flat NumPy + arrays (source, destination, frequency) sorted by source position. + Filters return lightweight views that share the vertex/edge store and + differ only in their boolean edge mask, so no filter ever copies the + graph. On top of that store it supports composable hard-constraint masks + (`vertex_pass_filter`, `frequency_filter`), pruning to complete paths, + exact path counting, and exhaustive/uniform/weighted generation. + """ + + def __init__(self): + self.start_vertices = {} + # Shared vertex/edge store; populated by load_sequences() or shared + # by views. + self.vertex_symbols = [] + self.vertex_positions = np.empty(0, dtype=np.int32) + self.vertex_index = {} + self.edge_src = np.empty(0, dtype=np.int32) + self.edge_dst = np.empty(0, dtype=np.int32) + self.edge_freq = np.empty(0, dtype=np.float64) + self.edge_pos = np.empty(0, dtype=np.int32) + self._position_slices = {} + self.edge_mask = np.empty(0, dtype=bool) + self._edge_lookup = None + self._out_edges = None + + # ------------------------------------------------------------------ + # Construction + + def load_sequences(self, sequences): + """ + Build the store from an iterable of (symbols, weight) pairs, where + symbols is an indexable sequence of hashable payloads. Parallel + edges accumulate their weights. + """ + vertex_index = {} + vertex_symbols = [] + vertex_positions = [] + edge_weights = {} + + def intern(position, symbol): + key = (position, symbol) + vertex_id = vertex_index.get(key) + if vertex_id is None: + vertex_id = len(vertex_symbols) + vertex_index[key] = vertex_id + vertex_symbols.append(symbol) + vertex_positions.append(position) + return vertex_id + + for symbols, weight in sequences: + n = len(symbols) + previous = intern(0, symbols[0]) + for j in range(1, n): + current = intern(j, symbols[j]) + edge = (previous, current) + edge_weights[edge] = edge_weights.get(edge, 0) + weight + previous = current + + self.vertex_index = vertex_index + self.vertex_symbols = vertex_symbols + self.vertex_positions = np.array(vertex_positions, dtype=np.int32) + nedges = len(edge_weights) + src = np.empty(nedges, dtype=np.int32) + dst = np.empty(nedges, dtype=np.int32) + freq = np.empty(nedges, dtype=np.float64) + for i, ((s, d), f) in enumerate(edge_weights.items()): + src[i] = s + dst[i] = d + freq[i] = f + # Sort edges by source position so pruning can sweep positions + # back-to-front over contiguous slices. + pos = self.vertex_positions[src] if nedges else np.empty(0, np.int32) + order = np.argsort(pos, kind='stable') + self.edge_src = src[order] + self.edge_dst = dst[order] + self.edge_freq = freq[order] + self.edge_pos = pos[order] + self._build_position_slices() + self.edge_mask = np.ones(nedges, dtype=bool) + self._invalidate_caches() + self.set_start_vertices() + + def _cache_arrays(self): + """The array half of a picklable snapshot; symbols are the caller's.""" + return { + 'vertex_positions': self.vertex_positions, + 'edge_src': self.edge_src, + 'edge_dst': self.edge_dst, + 'edge_freq': self.edge_freq, + 'edge_pos': self.edge_pos, + } + + def _restore_arrays(self, state, vertex_symbols): + """Rebuild the store from _cache_arrays() output plus its symbols.""" + self.vertex_symbols = vertex_symbols + self.vertex_positions = state['vertex_positions'] + self.vertex_index = { + (int(position), symbol): vertex_id + for vertex_id, (position, symbol) + in enumerate(zip(self.vertex_positions, vertex_symbols))} + self.edge_src = state['edge_src'] + self.edge_dst = state['edge_dst'] + self.edge_freq = state['edge_freq'] + self.edge_pos = state['edge_pos'] + self._build_position_slices() + self.edge_mask = np.ones(len(self.edge_src), dtype=bool) + self._invalidate_caches() + self.set_start_vertices() + + def _build_position_slices(self): + self._position_slices = {} + if len(self.edge_pos) == 0: + return + positions = np.unique(self.edge_pos) + starts = np.searchsorted(self.edge_pos, positions, side='left') + ends = np.searchsorted(self.edge_pos, positions, side='right') + for position, start, end in zip(positions, starts, ends): + self._position_slices[int(position)] = slice(int(start), int(end)) + + def _view(self, edge_mask): + """A graph sharing this graph's store with a different edge mask.""" + view = type(self).__new__(type(self)) + view.start_vertices = {} + view.vertex_symbols = self.vertex_symbols + view.vertex_positions = self.vertex_positions + view.vertex_index = self.vertex_index + view.edge_src = self.edge_src + view.edge_dst = self.edge_dst + view.edge_freq = self.edge_freq + view.edge_pos = self.edge_pos + view._position_slices = self._position_slices + view.edge_mask = edge_mask + view._invalidate_caches() + self._share_domain_state(view) + return view + + def _share_domain_state(self, view): + """Hook for subclasses to carry their own state onto filter views.""" + + def _invalidate_caches(self): + self._edge_lookup = None + self._out_edges = None + + # ------------------------------------------------------------------ + # Derived structures (cached per view) + + def _edges(self): + """{(src_id, dst_id): frequency} over active edges.""" + if self._edge_lookup is None: + active = np.flatnonzero(self.edge_mask) + self._edge_lookup = { + (int(self.edge_src[i]), int(self.edge_dst[i])): + float(self.edge_freq[i]) + for i in active} + return self._edge_lookup + + def _adjacency(self): + """{src_id: [dst_id, ...]} over active edges.""" + if self._out_edges is None: + adjacency = defaultdict(list) + for i in np.flatnonzero(self.edge_mask): + adjacency[int(self.edge_src[i])].append(int(self.edge_dst[i])) + self._out_edges = dict(adjacency) + return self._out_edges + + def _vertex_id(self, position, symbol): + return self.vertex_index.get((position, symbol)) + + # ------------------------------------------------------------------ + # Mapping-style compatibility (cold paths: display, dumps, limits) + + def __len__(self): + return len(self._adjacency()) + + def __contains__(self, key): + vertex_id = self._vertex_id(key.position, key.value) + return vertex_id is not None and vertex_id in self._adjacency() + + def keys(self): + return [Vertex(int(self.vertex_positions[vertex_id]), + self.vertex_symbols[vertex_id]) + for vertex_id in self._adjacency()] + + def items(self): + for vertex_id, children in self._adjacency().items(): + key = Vertex(int(self.vertex_positions[vertex_id]), + self.vertex_symbols[vertex_id]) + edges = self._edges() + nextkeys = { + Vertex(int(self.vertex_positions[child]), + self.vertex_symbols[child]): + edges[(vertex_id, child)] + for child in children} + yield key, nextkeys + + # ------------------------------------------------------------------ + # Public interface + + def set_start_vertices(self, reference_sequence=None, fields=None): + adjacency = self._adjacency() + self.start_vertices = { + Vertex(0, self.vertex_symbols[vertex_id]): 0 + for vertex_id in adjacency + if self.vertex_positions[vertex_id] == 0} + + def get_frequencies(self, reference_sequence): + frequencies = {} + edges = self._edges() + for position in range(len(reference_sequence) - 1): + source = self._vertex_id(position, reference_sequence[position]) + target = self._vertex_id( + position + 1, reference_sequence[position + 1]) + if source is None or target is None: + frequencies[position] = 0 + else: + frequencies[position] = edges.get((source, target), 0) + return frequencies + + def edge_mask_filter(self, mask): + """View keeping only active edges that also pass the given mask.""" + if len(self.edge_mask): + mask = self.edge_mask & mask + else: + mask = self.edge_mask.copy() + return self._view(mask) + + def vertex_pass_filter(self, vertex_pass, require_dst=True): + """ + View keeping edges whose source vertex (and, unless require_dst is + False, destination vertex) passes the given boolean array. + """ + if len(self.edge_mask): + mask = self.edge_mask & vertex_pass[self.edge_src] + if require_dst: + mask = mask & vertex_pass[self.edge_dst] + else: + mask = self.edge_mask.copy() + return self._view(mask) + + def vertex_predicate_filter(self, predicate, require_dst=True): + """vertex_pass_filter with the array built from predicate(position, symbol).""" + vertex_pass = np.fromiter( + (predicate(int(position), symbol) + for position, symbol + in zip(self.vertex_positions, self.vertex_symbols)), + dtype=bool, count=len(self.vertex_symbols)) + return self.vertex_pass_filter(vertex_pass, require_dst=require_dst) + + def frequency_filter(self, reference_sequence, lower, upper, kind='dev'): + # Concentric search widens bands as 2**k without bound; clamp so the + # arbitrary-precision ints survive the conversion to float64. + lower = float(max(min(lower, 1e300), -1e300)) + upper = float(max(min(upper, 1e300), -1e300)) + nref = len(reference_sequence) + max_position = int(self.edge_pos[-1]) if len(self.edge_pos) else -1 + lower_by_position = np.full(max_position + 2, np.inf) + upper_by_position = np.full(max_position + 2, -np.inf) + if kind == 'dev': + frequencies = self.get_frequencies(reference_sequence) + for position, frequency in frequencies.items(): + if position <= max_position: + lower_by_position[position] = frequency - lower + upper_by_position[position] = frequency + upper + elif kind == 'limit': + limit = min(nref - 1, max_position + 1) + lower_by_position[:limit] = lower + upper_by_position[:limit] = upper + if len(self.edge_pos): + edge_lower = lower_by_position[self.edge_pos] + edge_upper = upper_by_position[self.edge_pos] + mask = (self.edge_mask + & (self.edge_freq >= edge_lower) + & (self.edge_freq <= edge_upper)) + else: + mask = self.edge_mask.copy() + result = self._view(mask)._prune_in_place(nref - 1) + result.set_start_vertices() + return result + + def prune(self, maxpos): + """ + Remove paths that can not be completed. + """ + return self._view(self.edge_mask.copy())._prune_in_place(maxpos) + + def _prune_in_place(self, maxpos): + """ + Keep only edges on a path reaching position maxpos: one backward + sweep over positions instead of the old copy-until-fixpoint pass. + An edge survives if its destination is at maxpos or its destination + has a surviving outgoing edge. + """ + mask = self.edge_mask + if len(mask) == 0: + return self + mask[self.edge_pos >= maxpos] = False + alive = self.vertex_positions == maxpos + for position in range(maxpos - 1, -1, -1): + edge_slice = self._position_slices.get(position) + if edge_slice is None: + continue + keep = mask[edge_slice] & alive[self.edge_dst[edge_slice]] + mask[edge_slice] = keep + alive[self.edge_src[edge_slice][keep]] = True + self._invalidate_caches() + return self + + def _resolve_start_vertices(self, start_vertices, adjacency): + """[(symbol value, vertex id or None if not in the graph), ...]""" + starts = [] + for key in start_vertices: + vertex_id = self._vertex_id(key.position, key.value) + if vertex_id is not None and vertex_id in adjacency: + starts.append((key.value, vertex_id)) + else: + starts.append((key.value, None)) + return starts + + def _completion_counts(self, weighted=False): + """ + Per-vertex number of complete paths (or, when weighted, total path + weight as the product of edge frequencies) from the vertex to any + terminal vertex, over active edges. A vertex with no active outgoing + edges is a terminal and counts as one path of weight 1, matching + generate()'s yield condition. + """ + adjacency = self._adjacency() + vertices = set(adjacency) + for children in adjacency.values(): + vertices.update(children) + edges = self._edges() if weighted else None + counts = {} + for vertex in sorted( + vertices, key=lambda n: int(self.vertex_positions[n]), + reverse=True): + children = adjacency.get(vertex) + if not children: + counts[vertex] = 1.0 if weighted else 1 + elif weighted: + counts[vertex] = sum( + edges[(vertex, child)] * counts[child] + for child in children) + else: + counts[vertex] = sum(counts[child] for child in children) + return counts + + def count_paths(self, start_vertices=None): + """ + Exact number of sequences generate() would yield, without + generating them. Start vertices that are not part of the graph yield a + single one-symbol sequence and therefore count as one path. + """ + if start_vertices is None: + start_vertices = self.start_vertices + counts = self._completion_counts() + total = 0 + for _, vertex_id in self._resolve_start_vertices(start_vertices, + self._adjacency()): + total += 1 if vertex_id is None else counts[vertex_id] + return total + + def generate(self, start_vertices=None, mode='exhaustive', rng=random): + """ + Yield complete symbol sequences from the (filtered) graph. + + Modes: + 'exhaustive' — depth-first traversal with shuffled edge order + (the classic behavior). Complete, but sequence order is + biased toward low-branching regions of the graph. + 'uniform' — every sequence exactly once, in unbiased random + order (uniform sampling without replacement). + 'weighted' — every sequence exactly once, sampled without + replacement with probability proportional to the product + of its transition frequencies, so the highest-weight + sequences tend to arrive first. + """ + if mode == 'uniform': + return self._generate_sampled(start_vertices, weighted=False, rng=rng) + if mode == 'weighted': + return self._generate_sampled(start_vertices, weighted=True, rng=rng) + if mode != 'exhaustive': + raise ValueError(f"Unknown generation mode {mode!r}") + return self._generate_exhaustive(start_vertices, rng) + + def _generate_exhaustive(self, start_vertices, rng): + if start_vertices is None: + start_vertices = self.start_vertices + if len(self) == 0: + raise Exception('EmptyGraphError') + adjacency = self._adjacency() + symbols = self.vertex_symbols + + starts = self._resolve_start_vertices(start_vertices, adjacency) + rng.shuffle(starts) + + for start_value, start_id in starts: + if start_id is None: + yield (start_value,) + continue + path = [start_id] + children = adjacency[start_id][:] + rng.shuffle(children) + stack = [iter(children)] + while stack: + vertex_id = next(stack[-1], _SENTINEL) + if vertex_id is _SENTINEL: + stack.pop() + path.pop() + continue + children = adjacency.get(vertex_id) + if children: + path.append(vertex_id) + children = children[:] + rng.shuffle(children) + stack.append(iter(children)) + else: + yield tuple(symbols[i] for i in path) + ( + symbols[vertex_id],) + + def _generate_sampled(self, start_vertices, weighted, rng): + """ + Sampling without replacement over complete paths. + + The completion counts/weights below a vertex are shared by every + prefix that reaches it, so a drawn path must only be removed for + its own prefix. Removed paths are therefore tracked in a prefix + trie: during a draw, the available mass under child c is the + vertex's total (count or weight) minus what the trie records as + already drawn through this exact prefix. Integer counts guarantee + each path is yielded exactly once and the generator terminates; + float weights only bias the selection order in weighted mode. + Memory grows with the number of sequences drawn (one trie chain + per draw); use exhaustive mode for memory-free full drains. + """ + if start_vertices is None: + start_vertices = self.start_vertices + if len(self) == 0: + raise Exception('EmptyGraphError') + adjacency = self._adjacency() + symbols = self.vertex_symbols + edges = self._edges() + counts = self._completion_counts() + weights = self._completion_counts(weighted=True) if weighted else None + + starts = self._resolve_start_vertices(start_vertices, adjacency) + # Trie vertex: [removed_count, removed_weight, children_by_dag_vertex]. + trie_root = [0, 0.0, {}] + singletons_drawn = set() + + def available(trie_children, vertex_id): + removed = trie_children.get(vertex_id) + return counts[vertex_id] - (removed[0] if removed else 0) + + while True: + root_children = trie_root[2] + live = [] + for index, (value, vertex_id) in enumerate(starts): + if vertex_id is None: + if index not in singletons_drawn: + live.append((index, value, None, 1, 1.0)) + else: + avail = available(root_children, vertex_id) + if avail > 0: + removed = root_children.get(vertex_id) + mass = ((weights[vertex_id] + - (removed[1] if removed else 0.0)) + if weighted else float(avail)) + live.append((index, value, vertex_id, avail, mass)) + if not live: + return + + if weighted: + total_mass = sum(entry[4] for entry in live) + pick = rng.random() * max(total_mass, 0.0) + chosen = live[-1] + for entry in live: + pick -= entry[4] + if pick <= 0: + chosen = entry + break + else: + pick = rng.randrange(sum(entry[3] for entry in live)) + chosen = live[-1] + for entry in live: + pick -= entry[3] + if pick < 0: + chosen = entry + break + index, value, vertex_id = chosen[0], chosen[1], chosen[2] + if vertex_id is None: + singletons_drawn.add(index) + yield (value,) + continue + + path = [vertex_id] + current = vertex_id + trie_cursor = trie_root[2].get(vertex_id) + while True: + children = adjacency.get(current) + if not children: + break + trie_children = trie_cursor[2] if trie_cursor else {} + live_children = [] + for child in children: + avail = available(trie_children, child) + if avail <= 0: + continue + if weighted: + removed = trie_children.get(child) + mass = edges[(current, child)] * ( + weights[child] + - (removed[1] if removed else 0.0)) + live_children.append((child, avail, mass)) + else: + live_children.append((child, avail, float(avail))) + if weighted: + total_mass = sum(mass for _, _, mass in live_children) + pick = rng.random() * max(total_mass, 0.0) + current = live_children[-1][0] + for child, _, mass in live_children: + pick -= mass + if pick <= 0: + current = child + break + else: + pick = rng.randrange( + sum(avail for _, avail, _ in live_children)) + current = live_children[-1][0] + for child, avail, _ in live_children: + pick -= avail + if pick < 0: + current = child + break + path.append(current) + trie_cursor = (trie_cursor[2].get(current) + if trie_cursor else None) + + # Record the drawn path in the trie: below position i the + # removed suffix weight is the product of the remaining edge + # frequencies (1 at the terminal). + suffix_weights = [1.0] * len(path) + for i in range(len(path) - 2, -1, -1): + suffix_weights[i] = (edges[(path[i], path[i + 1])] + * suffix_weights[i + 1]) + cursor = trie_root + for vertex, suffix_weight in zip(path, suffix_weights): + child = cursor[2].get(vertex) + if child is None: + child = [0, 0.0, {}] + cursor[2][vertex] = child + child[0] += 1 + child[1] += suffix_weight + cursor = child + yield tuple(symbols[i] for i in path) + + def display(self): + for key, nextkeys in sorted(self.items(), key=lambda x: x): + print('***', key.position, key.value) + for nextkey, frequency in nextkeys.items(): + print(nextkey.value, frequency) diff --git a/wuggy/utilities/segmentgraph.py b/wuggy/utilities/segmentgraph.py new file mode 100644 index 0000000..5529348 --- /dev/null +++ b/wuggy/utilities/segmentgraph.py @@ -0,0 +1,134 @@ +import random +from collections import defaultdict + +from .positionalgraph import PositionalGraph, Vertex + +__all__ = ['SegmentGraph', 'Vertex', 'CACHE_FORMAT_VERSION'] + +# Bump when the cached on-disk representation changes shape. +CACHE_FORMAT_VERSION = 2 + + +class SegmentGraph(PositionalGraph): + """ + A PositionalGraph whose symbols are linguistic segments from a language + plugin. This layer owns everything that knows about language: parsing a + plugin data file, filters expressed over segment fields, limit + frequencies over named fields, and reconstructing plugin namedtuple + segments when a cached graph is loaded. All graph mechanics — interning, + edge arrays, masks, pruning, counting, generation — live in the kernel. + """ + + def __init__(self, language_plugin, data=None, encoding='utf-8', + size=100, cutoff=1, token=False): + super().__init__() + self.language_plugin = language_plugin + try: + self.hidden_sequence = self.language_plugin.hidden_sequence + except AttributeError: + self.hidden_sequence = False + self.limit_frequencies = {} + if data is not None: + self.load(data, size=size, cutoff=cutoff, token=token) + + # The kernel stores payloads as vertex_symbols; keep the segment-named + # accessor for callers of the pre-extraction attribute. + @property + def vertex_segments(self): + return self.vertex_symbols + + def _share_domain_state(self, view): + view.language_plugin = self.language_plugin + view.hidden_sequence = self.hidden_sequence + view.limit_frequencies = {} + + # ------------------------------------------------------------------ + # Construction + + def load(self, datafile, size=100, cutoff=1, token=False): + separator = self.language_plugin.separator + transform = self.language_plugin.transform + + def parsed_sequences(): + for line in datafile: + fields = line.strip('\n\t').split(separator) + reference, input_sequence, frequency = fields + frequency = float(frequency) if token == True else 1 + sequence = transform(input_sequence, frequency) + if frequency >= cutoff and ( + size >= 100 or random.randint(1, 100) <= size): + yield sequence.representation, sequence.frequency + + self.load_sequences(parsed_sequences()) + datafile.close() + + def to_cache_state(self): + """ + Picklable snapshot of the parsed store. Segments are flattened to + plain tuples because plugin namedtuple classes are class attributes + and cannot be pickled by reference. + """ + state = { + 'version': CACHE_FORMAT_VERSION, + 'vertex_tuples': [tuple(segment) + for segment in self.vertex_symbols], + } + state.update(self._cache_arrays()) + return state + + @classmethod + def from_cache_state(cls, language_plugin, state): + if state.get('version') != CACHE_FORMAT_VERSION: + raise ValueError('incompatible cache format') + graph = cls(language_plugin) + nfields = len(language_plugin.Segment._fields) + segments = [] + for values in state['vertex_tuples']: + segment_cls = (language_plugin.Segment + if len(values) == nfields + else language_plugin.SegmentH) + segments.append(segment_cls(*values)) + graph._restore_arrays(state, segments) + return graph + + # ------------------------------------------------------------------ + # Segment-aware filters and limits + + def segmentset_filter(self, reference_sequence, segmentset): + segmentset = segmentset.union(set(('^', '$'))) + result = self.vertex_predicate_filter( + lambda position, segment: segment.letters in segmentset) + result._prune_in_place(len(reference_sequence) - 1) + result.set_start_vertices() + return result + + def attribute_filter(self, reference_sequence, attribute): + nref = len(reference_sequence) + if type(reference_sequence[0]) == self.language_plugin.Segment: + def reference_value(position): + return getattr(reference_sequence[position], attribute) + else: + def reference_value(position): + return reference_sequence[position] + return self.vertex_predicate_filter( + lambda position, segment: ( + position < nref + and getattr(segment, attribute) == reference_value(position)), + require_dst=False) + + def build_limit_frequencies(self, fields): + limits = defaultdict(dict) + for key, nextkeys in self.items(): + position, value = key + subkey_a = (position, tuple( + getattr(value, field) for field in fields)) + for nextkey, frequency in nextkeys.items(): + position, value = nextkey + subkey_b = (position, tuple( + getattr(value, field) for field in fields)) + subkey = (subkey_a, subkey_b) + minfrequency = limits[subkey].get('min', frequency) + limits[subkey]['min'] = min(minfrequency, frequency) + maxfrequency = limits[subkey].get('max', frequency) + limits[subkey]['max'] = max(maxfrequency, frequency) + self.limit_frequencies[tuple(fields)] = limits