diff --git a/AGENTS.md b/AGENTS.md index d49d4381..aeca2f2f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,6 +119,15 @@ uv run sphinx-build -b html docs dist/docs # vocabulary (maiden markers, ambiguous acronyms). An alternation # matching no key fails as undeclared -- add it, or record it in # _NOT_A_VOCABULARY_COPY if it copies nothing. +# A ledger may also carry [[never]] entries -- shapes that must stay +# unexplained, documented in tools/differential/README.md. compare.py's +# validate_exclusions checks them at startup and +# tests/v2/test_ledger_guards.py records what each one silences -- +# which rules would claim it with exclusions off, and how much corpus it +# captures -- in _EXCLUSION_EFFECT. That IS an enrollment: an entry +# added, or its name_regex/fields/examples edited, moves the record and +# must be re-recorded deliberately, the same forcing function +# _CORPUS_CLAIMS applies to rules. (A `why`-only edit does not move it.) # 9. Open the next cycle's VERSION: bump VERSION in nameparser/_version.py to # the minor now being worked, and set PRE_RELEASE = 'dev'. The tree then says # what it is building rather than what it last shipped -- docs/conf.py reads diff --git a/tests/v2/_differential_fixtures.py b/tests/v2/_differential_fixtures.py index 2e32975f..2e9da482 100644 --- a/tests/v2/_differential_fixtures.py +++ b/tests/v2/_differential_fixtures.py @@ -77,6 +77,16 @@ def _rules(ledger: Path) -> list[dict]: ledger.read_text(encoding="utf-8")).get("change", []) +def _exclusions(ledger: Path) -> list[dict]: + """The [[never]] table of one ledger. + + Same .get default as _rules and for the same reason: a ledger with + no exclusions is the normal case, not a malformed file. + """ + return tomllib.loads( + ledger.read_text(encoding="utf-8")).get("never", []) + + def load_tool(stem: str) -> ModuleType: """A module from tools/differential/, loaded by path. diff --git a/tests/v2/test_differential.py b/tests/v2/test_differential.py index a26d20d4..ce45db31 100644 --- a/tests/v2/test_differential.py +++ b/tests/v2/test_differential.py @@ -295,13 +295,13 @@ def test_default_baseline_has_a_ledger_and_nothing_else_in_it() -> None: run abort -- and it would also make the carve-out inert, since it would name a file nothing iterates over. - And it must define nothing at the top level except `change`. This is - the one ledger allowed to be empty, so a mistyped table header -- - `[[changes]]`, `[[rules]]` -- reads as a legitimately empty open - cycle everywhere instead of as a broken file: every sweep gets zero - rules and passes, while the author believes they shipped a rule. - The other ledgers are protected by having to be non-empty; this one - needs saying out loud. + And it must define nothing at the top level except `change` and + `never`, the two keys anything reads. This is the one ledger allowed + to be empty, so a mistyped table header -- `[[changes]]`, `[[rules]]`, + `[[nevr]]` -- reads as a legitimately empty open cycle everywhere + instead of as a broken file: every sweep gets zero rules and passes, + while the author believes they shipped a rule. The other ledgers are + protected by having to be non-empty; this one needs saying out loud. """ import tomllib open_cycle = _TOOLS / f"expected_since_{compare.DEFAULT_BASELINE}.toml" @@ -310,10 +310,11 @@ def test_default_baseline_has_a_ledger_and_nothing_else_in_it() -> None: f"{open_cycle.name} does not exist; a bare compare.py run would " f"hard-error, and the empty-ledger carve-out would be inert") keys = set(tomllib.loads(open_cycle.read_text(encoding="utf-8"))) - assert keys <= {"change"}, ( - f"{open_cycle.name} defines {sorted(keys - {'change'})} at the top " - f"level. Only `change` is read, so anything else is a typo that " - f"would read as an empty ledger rather than as a broken one") + assert keys <= {"change", "never"}, ( + f"{open_cycle.name} defines {sorted(keys - {'change', 'never'})} at " + f"the top level. Only `change` and `never` are read, so anything " + f"else is a typo that would read as an empty ledger rather than as " + f"a broken one") def test_validate_rules_accepts_the_shipped_ledgers() -> None: @@ -723,3 +724,104 @@ def test_main_aborts_on_a_corpus_with_no_floor( _run_main(tmp_path, monkeypatch, '[[change]]\nissue = "x"\nname_regex = "ZZZ"\n', _SAME_FACADE, floor=None) + + +# The exclusion grammar. Every row is a way a [[never]] entry can look +# correct and protect nothing -- the same failure the rules' own +# validator exists for, pointed at the section that DISABLES +# classification instead of the one that performs it. +@pytest.mark.parametrize("entry,expect", [ + ({}, "no string 'why'"), + ({"why": ""}, "no string 'why'"), + ({"why": "x"}, "no string 'name_regex'"), + ({"why": "x", "name_regex": "Smith("}, "invalid 'name_regex'"), + ({"why": "x", "name_regex": "a"}, "no 'examples'"), + ({"why": "x", "name_regex": "a", "examples": []}, "no 'examples'"), + ({"why": "x", "name_regex": "a", "examples": "b"}, "not a list of strings"), + ({"why": "x", "name_regex": "a", "examples": ["a", 1]}, + "not a list of strings"), + # an example the entry does not actually protect + ({"why": "x", "name_regex": "zzz", "examples": ["John Smith"]}, + "does not match its own"), + # a misspelled key deletes half the declaration, exactly as for rules + ({"why": "x", "name_regex": "a", "examples": ["a"], "reason": "b"}, + "unknown key"), + # would silence the entire ledger + ({"why": "x", "name_regex": ".", "examples": ["a"]}, + "matches every one of"), +]) +def test_validate_exclusions_rejects_an_entry_that_protects_nothing( + entry: dict, expect: str) -> None: + with pytest.raises(SystemExit, match=expect): + compare.validate_exclusions([entry], "expected_since_1.4.0.toml") + + +def test_validate_exclusions_accepts_the_shipped_entries() -> None: + """The guards above must not be so strict they reject real entries.""" + import tomllib + for ledger in sorted(_TOOLS.glob("expected_since_*.toml")): + parsed = tomllib.loads(ledger.read_text(encoding="utf-8")) + compare.validate_exclusions(parsed.get("never", []), ledger.name) + + +def test_classify_refuses_an_excluded_shape() -> None: + """The whole point: an excluded name reports UNEXPLAINED however + many rules would otherwise claim it. Two do, for the shape this + was built for -- fix(comma-family) on file order, and the + fields-only fix(suffix-routing) which has no name_regex at all and + so reaches every name.""" + rules = [{"issue": "broad", "name_regex": ","}, + {"issue": "broader", "fields": ["given", "suffix"]}] + never = [{"why": "parity", "name_regex": r"(?i)\bph\.\s*d\.\s*$", + "examples": ["John Smith, Ph. D."]}] + assert compare.classify("John Smith, Ph. D.", {"suffix"}, rules) == "broad" + assert compare.classify( + "John Smith, Ph. D.", {"suffix"}, rules, never) is None + # a name the exclusion does not cover is unaffected + assert compare.classify("Smith, Dr.", {"suffix"}, rules, never) == "broad" + + +@pytest.mark.parametrize("entry,expect", [ + ({"why": "x", "name_regex": "a", "examples": ["a"], "fields": "given"}, + "not a list of strings"), + ({"why": "x", "name_regex": "a", "examples": ["a"], "fields": []}, + "empty 'fields'"), + ({"why": "x", "name_regex": "a", "examples": ["a"], "fields": ["nope"]}, + "not roles"), + # the facade's vocabulary is not the role vocabulary + ({"why": "x", "name_regex": "a", "examples": ["a"], "fields": ["first"]}, + "not roles"), + # all seven means "any diff", which is what omitting the key does + ({"why": "x", "name_regex": "a", "examples": ["a"], + "fields": ["title", "given", "middle", "family", "suffix", + "nickname", "maiden"]}, "omit 'fields'"), +]) +def test_validate_exclusions_rejects_a_bad_fields_narrowing( + entry: dict, expect: str) -> None: + with pytest.raises(SystemExit, match=expect): + compare.validate_exclusions([entry], "expected_since_1.4.0.toml") + + +def test_an_excluded_shape_stays_classifiable_on_other_roles() -> None: + """The reason `fields` exists. ASCII parens mark nicknames, maiden + names, suffixes and credentials alike, so an exclusion that names + the nickname reading must not silence a suffix diff on the same + name. Such a diff would not be hidden -- an excluded name reports + UNEXPLAINED and exits non-zero -- but it could never be classified + as intended either, leaving an area under active development + permanently unexplainable.""" + rules = [{"issue": "catch-all", "fields": ["given", "suffix", + "nickname", "middle"]}] + never = [{"why": "ascii pairs were already handled in 1.4", + "name_regex": r"\w\s+\([^)]+\)\s+\w", + "fields": ["nickname", "middle"], + "examples": ["John (Jack) Kennedy"]}] + name = "Lon (Jr.) Williams" + # the reading the exclusion names is refused + assert compare.classify(name, {"nickname"}, rules, never) is None + assert compare.classify(name, {"middle"}, rules, never) is None + # a different reading of the same name is still classifiable + assert compare.classify(name, {"suffix"}, rules, never) == "catch-all" + # and a mixed diff is not a subset of the exclusion, so it survives + assert compare.classify( + name, {"nickname", "suffix"}, rules, never) == "catch-all" diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 7ad21966..dde31da7 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -26,9 +26,11 @@ class declares, which members an alternation offers. Those are exact Split from test_regex_sync.py, which shares none of this (#352). """ import hashlib +import itertools import json import re from pathlib import Path +from types import ModuleType from typing import NamedTuple import pytest @@ -47,8 +49,8 @@ class declares, which members an alternation offers. Those are exact GLUED_HONORIFICS, SUFFIX_ACRONYMS_AMBIGUOUS, SUFFIX_WORDS) from ._differential_fixtures import ( - _CORPUS_NAMES, _LEDGERS, _TOOLS, _UNCLASSIFIED_NAMES, _claimed, _rules, - _unclassified_names, load_tool) + _CORPUS_NAMES, _LEDGERS, _TOOLS, _UNCLASSIFIED_NAMES, _claimed, + _exclusions, _rules, _unclassified_names, load_tool) # The one sanctioned divergence between the differential rules' @@ -1115,3 +1117,197 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: f"_CORPUS_CLAIMS names ledgers that do not exist: " f"{sorted(set(_CORPUS_CLAIMS) - {L.name for L in _LEDGERS})}") + + +class _Excluded(NamedTuple): + """What a [[never]] entry silences, in the two dimensions that can + change under it.""" + #: corpus names its name_regex captures + captures: int + #: sha256[:12] of those names, so a regex swap holding the count + #: still fails -- the identity-free lesson _CORPUS_CLAIMS records + digest: str + #: rules that WOULD claim a protected reading of its examples, with + #: exclusions switched OFF. This is the #328 event: a rule widened + #: to reach a protected shape joins this tuple. + absorbed_by: tuple[str, ...] + + +#: What each exclusion silences today, keyed by its name_regex. +#: +#: The first version of this guard asked whether a rule claims a +#: protected shape WITH exclusions active. It could not fail: classify() +#: consults exclusions first and returns None, and the guard only asked +#: about subsets the exclusion covers, so the answer was None by +#: construction. Measured: prepending a catch-all rule, and deleting +#: every rule in the ledger, both left it green across all 387 subsets. +#: It was the tenth inert measurement recorded in this tree. +#: +#: So ask with exclusions OFF, and record the answer. Then a rule +#: widened to reach a protected shape changes `absorbed_by` and demands +#: a decision -- which is the event #328 is about, and the one the +#: harness cannot report because the exclusion (correctly) hides it. +#: +#: `captures` and `digest` cover the opposite direction, which nothing +#: else watched: an exclusion widened by name_regex silences real +#: classifications, and BEFORE this record CI stayed green while the +#: release gate broke. Measured then: dropping the Ph. D. entry's +#: Latin anchor left the whole suite passing and took a bare run to +#: unexplained: 1. Measured now: the same edit fails this pin -- the +#: record is keyed by name_regex, so editing one is exactly what it +#: catches -- while the gate still goes to unexplained: 1. +#: +#: One limit worth naming: `absorbed_by` records only the FIRST rule +#: matching each subset, so a rule appended behind one that already +#: answers those subsets is invisible here. A rule reaching a +#: protected READING is caught; a rule shadowed by an existing one on +#: every subset it claims is not. +_EXCLUSION_EFFECT: dict[str, _Excluded] = { + "(?i)^[\\u0000-\\u024f]*\\bph\\.\\s*d\\.\\s*$": + _Excluded(3, "5a12a8117651", + ("fix(comma-family)", "fix(suffix-routing)")), + '[\\w.]\\s+[("\'][^)"\']+[)"\']\\s+\\w': + _Excluded(13, "42d04d428edf", ()), +} + + +def _protectable_fields(compare: ModuleType) -> tuple[str, ...]: + """The universe both exclusion pins quantify over: every field a + rule's `fields` may name, which is what an exclusion's may name too. + + Not `Role` alone. validate_exclusions accepts `_ambiguities` -- a + SEGMENTATION-only diff is facade-identical, so it is the one name + that can classify one -- and a universe of the seven roles never + builds a subset containing it. Measured: a rule with + `fields = ["_ambiguities"]` and no `name_regex` claims the + ambiguity-only reading of every protected shape, which is the #328 + event in that dimension, and only the wider universe grows + `absorbed_by` and fails. + + It does NOT close the matching hole on the exclusion side: an entry + narrowing itself to `fields = ["_ambiguities"]` protects nothing + anyone would notice and passes both pins under either universe, + because `absorbed_by` is legitimately empty for the honest entry + too. Taking the universe from compare's own set at least keeps the + two from drifting apart as `_RULE_FIELDS` grows. + """ + return tuple(sorted(compare._RULE_FIELDS)) + + +def test_every_exclusion_silences_what_is_recorded() -> None: + """Both directions an exclusion can drift, recorded rather than + derived -- because a derivation from the same data always agrees + with itself, which is how the first version of this guard came to + be tautological. + + `absorbed_by` is asked with exclusions OFF. That is the only way to + see the #328 event at all: once an entry is in place the harness + reports nothing, correctly, so a rule widened to reach a protected + shape is invisible everywhere else. When this tuple grows, someone + has to decide whether the new rule is legitimate and the exclusion + is now doing real work, or whether the rule reached too far. + + `captures`/`digest` are the opposite drift. An over-wide exclusion + silences classifications a rule should make -- loud at release, + silent in CI, which is the wrong way round for something a push + can introduce. + """ + compare = load_tool("compare") + roles = _protectable_fields(compare) + actual: dict[str, _Excluded] = {} + for ledger in _LEDGERS: + rules = compare._sorted_rules(_rules(ledger)) + for entry in _exclusions(ledger): + captured = sorted(name for name in _CORPUS_NAMES + if re.search(entry["name_regex"], name)) + covered = entry.get("fields") + absorbed = set() + for example in entry["examples"]: + for size in range(1, len(roles) + 1): + for combo in itertools.combinations(roles, size): + diff = set(combo) + if covered is not None and not diff <= set(covered): + continue + claimed = compare.classify(example, diff, rules) + if claimed: + absorbed.add(claimed.split(")")[0] + ")") + actual[entry["name_regex"]] = _Excluded( + len(captured), + hashlib.sha256( + "\n".join(captured).encode("utf-8")).hexdigest()[:12], + tuple(sorted(absorbed))) + assert actual == _EXCLUSION_EFFECT, ( + f"what an exclusion silences has moved. Recorded " + f"{_EXCLUSION_EFFECT}, now {actual}. A grown `absorbed_by` means " + f"a rule now reaches a protected shape -- decide whether that " + f"rule is right before recording it. A changed captures/digest " + f"means the exclusion itself moved, which is loud at release " + f"and silent here until this fails.") + assert actual, ( + "no ledger declares a [[never]] entry, so this pin is passing " + "vacuously") + + +def test_a_fields_narrowing_actually_narrows_something() -> None: + """The other direction, which nothing else watches. + + test_every_exclusion_silences_what_is_recorded pins an entry's + reach by NAME -- how much corpus it captures, and which rules would + claim its examples. It says nothing about whether the `fields` + narrowing on top of that reach still leaves anything behind, and + the two failures are not symmetric: an over-wide `fields` silences + diffs a rule should explain, on names that are not examples and so + are looked at nowhere else. + + Note where this fails when `fields` is DELETED outright: on the + vacuity assert at the end, not on the per-entry assert below, since + a deleted key drops the entry from the loop entirely. That works + only while one entry carries `fields`. A second one would leave the + deletion green here -- caught instead by `absorbed_by` in the + recorded pin, which is then asked about every reading rather than + the three the key covers, and sees rules claim them. Measured: + deleting this entry's `fields` grows its `absorbed_by` from () to + ('fix(suffix-routing)',). + + Measured: deleting `fields = ["nickname", "middle"]` from the + ASCII-pairs entry passes every other check in this tree. The entry + then refuses ANY diff on the thirteen corpus names it captures -- + including 'Jenny (Johnson) Baker' and 'Lon (Jr.) Williams', whose + parens are a maiden name and a suffix, both under active + development. Nothing failed, because none of those names diffs + today and none of them is an example. + + So: an entry that bothers to name `fields` must leave something + behind. If no captured corpus name is still classifiable on a + reading outside them, the narrowing is not narrowing -- either it + was deleted, or it grew to cover everything the entry reaches. + """ + compare = load_tool("compare") + roles = _protectable_fields(compare) + checked = 0 + for ledger in _LEDGERS: + rules = compare._sorted_rules(_rules(ledger)) + never = _exclusions(ledger) + for entry in never: + covered = entry.get("fields") + if covered is None: + continue + checked += 1 + captured = [name for name in _CORPUS_NAMES + if re.search(entry["name_regex"], name)] + survives = [ + (name, sorted(combo)) + for name in captured + for size in (1, 2) + for combo in itertools.combinations(roles, size) + if not set(combo) <= set(covered) + and compare.classify(name, set(combo), rules, never)] + assert survives, ( + f"{ledger.name}: the entry for {entry['name_regex']!r} names " + f"fields={covered}, but no corpus name it captures is still " + f"classifiable on any reading outside them. It captures " + f"{len(captured)} names, so the narrowing has stopped " + f"narrowing -- check it was not deleted or widened to cover " + f"the whole entry.") + assert checked, ( + "no exclusion declares `fields`, so this pin is passing vacuously") diff --git a/tools/differential/README.md b/tools/differential/README.md index b502711d..0781e1a8 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -277,6 +277,62 @@ suffix-delimiter rendering, which only fires under a non-default matching the family documented in `tests/v2/cases.py`, so the rule is ready the moment a matching string is added to the corpus. +### Shapes that must never be explained (`[[never]]`) + +A `[[change]]` rule says "this diff is intended, and here is what +changed". There was no rule meaning "whatever happens here is a +regression", so two comments in `expected_since_1.4.0.toml` promised +exactly that in prose while rules in the same file claimed those +shapes anyway -- both false from the day they were written until #328 +found them. A `[[never]]` entry is that promise made executable: it +names a shape that must stay unexplained. + +An entry needs `why` -- an exclusion nobody can justify is one nobody +can safely delete -- plus `name_regex` and `examples`. The examples are +required, not decoration: a protected shape need not appear in any +corpus, so the entry has to carry its own test data. `fields` is +optional and narrows WHICH READING is protected, by the same subset +test the rules use. + +That last key earns its keep on the ASCII pairs. Parens mark nicknames, +maiden names, suffixes and credentials alike, and no regex tells them +apart, so a name-only exclusion for the nickname promise would also +silence every diff on `Jenny (Johnson) Baker` and `Lon (Jr.) Williams`, +whose parens are a maiden name and a suffix. Nothing is hidden by that +-- an excluded name reports UNEXPLAINED, which exits non-zero -- but +those names become permanently unexplainable, so an intended change +there could never be recorded. Typographic delimiters carry no such +ambiguity, which is why `feat(#273)`'s own rule can be a bare character +class and its exclusion cannot. + +The ASCII-pairs entry is also narrowed by SHAPE, to a delimited run +medial to the name. That is what keeps the trailing credentials +(`Andrew Perkins (JD)` and its kin) out, and it is the limit of the +promise: a leading or trailing ASCII nickname pair is still unguarded, +for want of a regex that tells one from a credential. + +`classify()` consults exclusions BEFORE the rules, and a match returns +`None`, so an excluded shape reports UNEXPLAINED however many rules +would claim it. That order is also what makes exclusions monotone: an +entry only ever removes a name from classification, never moves it +between rules, so its blast radius is exactly the names it captures, on +the readings it names -- there is no rule ordering to reason about. + +`tests/v2/test_ledger_guards.py` records what each entry silences and +holds it there. Asking whether a rule claims a protected shape *with* +the exclusion active answers nothing -- `classify()` returns `None` +before the rules are reached, so the answer is `None` however the rules +change. The pin therefore asks with exclusions switched OFF and records +which rules WOULD claim each protected reading. A rule widened to reach +one changes that record and fails in CI, rather than being invisible +until someone reasons about it. It records the FIRST rule matching each +subset, so a rule shadowed on every subset it claims by one already +sitting ahead of it does not move the record; a rule reaching a +protected reading does. The same record carries the number and +digest of the corpus names an entry captures, which is the opposite +drift: an over-wide exclusion silences real classifications, and that +is loud at release but otherwise silent on a push. + ## What this gate does not cover The corpora run under the **default policy**, so any behavior gated diff --git a/tools/differential/compare.py b/tools/differential/compare.py index fc3a6f5d..e1083ee9 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -459,8 +459,134 @@ def validate_rules(rules: list[dict[str, object]], ledger: str) -> None: f"claimed every diff in the 1.4 ledger") +def validate_exclusions(entries: list[dict[str, object]], + ledger: str) -> None: + """Reject malformed [[never]] entries LOUDLY at startup. + + An exclusion is the absorption bug pointed the other way. A rule + that matches too widely turns a regression into a classified diff; + an exclusion that matches too widely turns a legitimate + classification into UNEXPLAINED, which reads as a catastrophic + regression rather than as a bad exclusion. So the checks mirror + validate_rules', with one addition: an entry whose `examples` do + not match its own `name_regex` protects nothing while looking + complete. Nothing else says so at startup, which is where a + silently-inert entry most needs saying. + + `fields` is optional and narrows WHICH READING is protected, the + same subset test the rules use. It exists because ASCII parens mark + nicknames, maiden names, suffixes and credentials alike -- a + name-only exclusion for the nickname promise would also silence + every diff on 'Jenny (Johnson) Baker' and 'Lon (Jr.) Williams', + whose parens are a maiden name and a suffix. That does not HIDE a + regression there -- an excluded name reports UNEXPLAINED, which + exits non-zero -- it makes those names permanently unexplainable, + so an intended change in an area under active development can + never be recorded and every release blocks on the same false + alarm. Typographic delimiters carry no such ambiguity, which is + why feat(#273)'s own rule can be a bare character class. + """ + allowed = {"why", "name_regex", "examples", "fields"} + for index, entry in enumerate(entries): + where = f"{ledger} exclusion #{index + 1}" + unknown = set(entry) - allowed + if unknown: + raise SystemExit( + f"{where} has unknown key(s) {sorted(unknown)}; expected " + f"{sorted(allowed)}. A misspelled key is silently ignored, " + f"which deletes whatever it was meant to declare.") + why = entry.get("why") + if not isinstance(why, str) or not why: + raise SystemExit( + f"{where} has no string 'why'. An exclusion nobody can " + f"justify is one nobody can safely delete.") + pattern = entry.get("name_regex") + if not isinstance(pattern, str) or not pattern: + raise SystemExit(f"{where} has no string 'name_regex'") + try: + compiled = re.compile(pattern) + except re.error as exc: + raise SystemExit( + f"{where} has an invalid 'name_regex' ({exc})") from None + if all(compiled.search(s) for s in _SENTINELS): + raise SystemExit( + f"{where}'s 'name_regex' matches every one of " + f"{list(_SENTINELS)} -- it would silence the whole ledger, " + f"reporting every diff as unexplained.") + examples = entry.get("examples") + if examples is None or examples == []: + raise SystemExit( + f"{where} has no 'examples'. They are the entry's test " + f"data: a protected shape need not be in any corpus, so " + f"nothing else can supply one.") + if (not isinstance(examples, list) + or not all(isinstance(e, str) for e in examples)): + raise SystemExit( + f"{where}'s 'examples' is not a list of strings") + stray = [e for e in examples if not compiled.search(e)] + if stray: + raise SystemExit( + f"{where} lists {stray} which does not match its own " + f"'name_regex' -- the entry would protect nothing while " + f"looking complete.") + if "fields" in entry: + fields = entry["fields"] + if not isinstance(fields, list) \ + or not all(isinstance(f, str) for f in fields): + raise SystemExit( + f"{where} has a 'fields' that is not a list of " + f"strings ({fields!r}); classify would ignore it and " + f"the entry would silence EVERY diff on a matching " + f"name, not the reading it names") + if not fields: + raise SystemExit( + f"{where} has an empty 'fields', which can never " + f"match any diff -- an exclusion that protects " + f"nothing") + bad = sorted(set(fields) - _RULE_FIELDS) + if bad: + raise SystemExit( + f"{where} names {bad} in 'fields', which are not " + f"roles; expected from {sorted(_RULE_FIELDS)}") + if set(V2_FIELDS) <= set(fields): + raise SystemExit( + f"{where} lists all seven roles in 'fields', which " + f"is what omitting the key already means. omit " + f"'fields' to exclude any diff on a matching name") + + def classify(name: str, diff_fields: set[str], - rules: list[dict[str, object]]) -> str | None: + rules: list[dict[str, object]], + exclusions: list[dict[str, object]] | None = None) -> str | None: + """Which rule explains this diff, or None if nothing does. + + Exclusions are consulted FIRST and win outright. They are the + ledger's way of saying a shape must never be explained, which the + rule vocabulary cannot express: a rule says "this diff is intended + and here is why", and there is no rule meaning "whatever happens + here is a regression". Two comments in expected_since_1.4.0.toml + promised exactly that in prose and could not keep it (#328). + + Consulting them first also makes them MONOTONE -- an exclusion only + ever removes a name from classification, never moves it between + rules -- so a new entry's blast radius is exactly the set of names + it captures, independent of rule order. + + An exclusion narrows by `name_regex` and optionally by `fields`, + exactly as a rule does. Without `fields` it refuses any diff on a + matching name; with them it refuses only the reading it names, so a + name whose parens mark a nickname to one rule and a suffix to + another stays classifiable on the reading the exclusion is not + about. + """ + for entry in exclusions or (): + pattern = entry.get("name_regex") + if isinstance(pattern, str) and not re.search(pattern, name): + continue + fields = entry.get("fields") + if isinstance(fields, list) and not diff_fields <= set(fields): + continue + return None for rule in rules: name_regex = rule.get("name_regex") if isinstance(name_regex, str) and not re.search(name_regex, name): @@ -492,9 +618,12 @@ def main() -> int: paths = ([Path(p) for p in args.corpus] if args.corpus else sorted(HERE.glob("corpus*.jsonl"))) ledger = _allowlist_for(baseline) - rules = tomllib.loads(ledger.read_text()).get("change", []) + parsed = tomllib.loads(ledger.read_text()) + rules = parsed.get("change", []) validate_rules(rules, ledger.name) rules = _sorted_rules(rules) + exclusions = parsed.get("never", []) + validate_exclusions(exclusions, ledger.name) # A glob that matches nothing must not read as "everything passed". # Comparing zero names would print 0 unexplained and exit 0 -- the # harness's own stated nightmare (see validate_rules), and a @@ -573,7 +702,7 @@ def main() -> int: if old["v2"].get(f, "") != new_v2.get(f, "")} if not diff: continue - issue = classify(name, diff, rules) + issue = classify(name, diff, rules, exclusions) if issue is None: unexplained.append( (name, old["facade"], new, old.get("v2", {}), new_v2)) diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 37d22b23..5647a26a 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -215,8 +215,12 @@ issue = "feat(#273) typographic nickname delimiters recognized by default" # 'John “Jack” Kennedy', 'Hans „Hansi“ Müller', 'Jean «Petit» Dupont': # v1 knew only straight quotes and parentheses, so a smart-quoted or # guillemet-wrapped nickname leaked into `middle` as literal text. -# Matches only when such a delimiter is actually present -- the ASCII -# pairs stay unclassified, since a diff there would be a regression. +# Matches only when such a delimiter is actually present. A nickname +# diff on the ASCII pairs would be a regression; the ASCII-pairs +# [[never]] entry at the end of this file is what refuses it, not this +# rule's silence (#328) -- and only for a MEDIAL pair, which is as far +# as a delimiter regex can reach without swallowing the trailing +# credentials. See that entry for what stays unguarded. name_regex = "[“”„«»「」『』（）]" fields = ["middle", "nickname"] @@ -389,28 +393,147 @@ fields = ["title", "given", "middle", "suffix"] # vocab tag (tests/v2/cases.py classification="parity" on both # 'phd_split' and 'suffix_comma_split_phd'; verified empirically, zero # diff against the 1.4 worker). Adding a suppression rule for it would -# risk masking a real regression in this exact shape, so it is -# intentionally left unclassified: if it ever starts diffing, the -# harness must fail. (The leading case above is a separate shape, which -# is why it needed its own anchored rule rather than a widened one.) +# risk masking a real regression in this exact shape, so the +# trailing-'Ph. D.' [[never]] entry below refuses it instead: any diff here +# reports UNEXPLAINED (#328). (The leading case above is a separate +# shape, which is why it needed its own anchored rule rather than a +# widened one.) # -# That last sentence does NOT hold today, and the scope of what is -# parity has to be read narrowly because of it. Parity is the two -# inputs named above, where the pair LEADS its run. A trailing pair -# after another suffix diffs on ORDER -- 'John Smith, Jr. Ph. D.' is -# 1.4.0 suffix 'Ph. D., Jr.' against 2.0's 'Jr. Ph. D.', because +# The scope of what is parity still has to be read narrowly. Parity is +# the two inputs named above, where the pair LEADS its run. A trailing +# pair after another suffix diffs on ORDER -- 'John Smith, Jr. Ph. D.' +# is 1.4.0 suffix 'Ph. D., Jr.' against 2.0's 'Jr. Ph. D.', because # fix_phd extracted the credential pre-parse and re-appended it while # 2.0 renders the tail as written (tests/v2/cases.py # 'suffix_comma_split_phd_after_another_suffix', -# classification="fix(credential-pair-order)"). Measured on a probe -# corpus: the run comes out unexplained 0, absorbed by -# fix(comma-family) below, whose name_regex is a bare comma and whose -# `fields` list contains `suffix`. So the shape is guarded by the case -# table and not by this file. Left that way deliberately rather than -# fixed in passing: classify() takes the FIRST matching rule and the -# sort has only two tiers (name_regex before fields-only, stable -# within a tier), so a narrower rule for this shape could only win by -# being written earlier in the file -- making file order load-bearing -# again, which is the thing the sort exists to prevent. Giving -# classify() a real specificity order is the fix, and it is a change -# to the harness rather than to a rule. +# classification="fix(credential-pair-order)"). That input is in the +# case table and in no corpus, so nothing is reported on it either +# way; note that the trailing-'Ph. D.' [[never]] entry's anchor +# captures it too, and would call its intended order diff a regression +# if a corpus ever gained it. Reconciling them then means a narrower +# `name_regex`, and only that: the two renderings differ in the ORDER +# of one suffix string and in nothing else, so the diff is `suffix` +# alone -- the one field the parity promise is about. Any `fields` +# list permissive enough to let the order diff classify would abandon +# the promise. Not a question to settle now, on a name the harness +# never sees. +# +# No ordering question arises here: exclusions are consulted before +# any rule, so this is independent of where it sits in the file. + +# Shapes that must never be explained. A [[change]] rule says "this +# diff is intended, and here is what changed"; there was no rule +# meaning "whatever happens here is a regression", so the two promises +# below could only be written as prose -- and both were false as +# written from the day they were written (the first shipped with the +# harness itself, the second three days later) until #328 found them. +# +# The two are not false alike, and the difference matters now that +# this file scopes the second promise to one reading. Both said +# "unclassified", unqualified, and both were claimed on SOME reading +# from day one. But only the Ph. D. promise was claimed on the reading +# it now protects; no rule has ever claimed the ASCII pairs' NICKNAME +# reading, then or today. That entry is a tripwire under no present +# tension, which is the point of setting it before the tension exists. +# +# classify() consults these before any rule, so a matching name +# reports UNEXPLAINED however many rules would claim it. +# +# `examples` is not decoration. A protected shape need not appear in +# any corpus -- 'John "Jack" Kennedy' does not -- so the entry carries +# its own test data, which tests/v2/test_ledger_guards.py runs against +# every non-empty subset of the fields a rule may name: the seven +# roles and the `_ambiguities` pseudo-field. An entry carrying +# `fields` is asked only about the subsets those cover -- 3 for the +# ASCII pairs, not 255 -- which is where the 387 in that file comes +# from. + +[[never]] +why = "trailing 'Ph. D.' split-token healing is PARITY, not a 2.0 change: v1 healed the adjacent pair too, so a diff here is a regression" +# Anchored to the END, and that is the whole design. The promise names +# two forms -- 'John Ph. D.' and 'John Smith, Ph. D.' -- so an +# exclusion keyed on the comma protects half of it while looking +# complete. The anchor also keeps clear of 'Ph. D. John Smith', the +# LEADING shape, which has its own fix(leading-credential) rule that +# this must not silence. No `fields`: any diff on this shape is a +# regression, which is what omitting the key means. +# +# One shape this deliberately over-reaches: 'John Smith, Jr. Ph. D.' +# still matches, and its ORDER diff (1.4's 'Ph. D., Jr.' against 2.x's +# 'Jr. Ph. D.') is an INTENDED change -- cases.py classifies it +# fix(credential-pair-order). No corpus contains it, so nothing is +# silenced today. Erring toward refusal is the safe direction: an +# UNEXPLAINED diff blocks a release for scrutiny, where a wrongly +# classified one passes silently. If a corpus ever gains a +# Ph. D. name, narrow this entry rather than widening a rule +# to claim it. +# +# Latin-anchored: an unrestricted trailing anchor also caught +# '田中さん, Ph. D.', whose diff is a real fix(cjk-comma-compound) +# classification -- measured, it took the gate to unexplained: 1. The +# cut is at U+0250, the same threshold _is_latin_only uses in +# compare.py, so it covers Latin-1 and Latin Extended-A and -B rather +# than bare ASCII: the split-token healing is script-independent, and +# 'José Smith, Ph. D.' is as much the protected shape as 'Jose Smith, +# Ph. D.' is. An ASCII-only class protected one and not the other. +name_regex = "(?i)^[\\u0000-\\u024f]*\\bph\\.\\s*d\\.\\s*$" +examples = ["John Ph. D.", "John Smith Ph. D.", "John Smith, Ph. D."] + +[[never]] +why = "feat(#273) recognizes TYPOGRAPHIC nickname delimiters; the ASCII pairs were already recognized in 1.4, so a NICKNAME diff on them is a regression" +# Narrowed twice, and both narrowings are load-bearing. +# +# By shape: a delimited run MEDIAL to the name -- flanked on the left +# by a word character or a period (so a middle initial counts: +# 'Cherice J. (Johnson) Williams') and on the right by a word +# character. A bare [("'] class reaches 47 corpus names. Measured by +# adding exactly that exclusion and running the gate, silencing all 47 +# costs THREE classifications -- '山田 太郎 (マイケル・ジャクソン)', +# '김, 민준씨 (Jimmy)', '김민준씨 (Jimmy)' -- because only those three +# diff against 1.4.0 at all. The other 44 would be pre-silenced: no +# diff today, and no way to report one tomorrow. Counting names +# overstates the first cost and hides the second. +# +# Of the 47, 8 are parenthesised credentials and 11 match on a bare +# apostrophe -- "Brian O'connor", "Harietta Keopuolani Nahi'ena'ena" -- +# which no delimiter regex can tell from a quote. +# +# The medial cut is about POSITION, not about credentials. Widening to +# leading and trailing runs takes the shape from 13 corpus names to +# 34, and 20 of the 21 it gains put the delimited run last. Only 8 of +# those are credentials; 6 are trailing NICKNAMES -- 'Franklin, +# Benjamin (Ben)', 'Rev John A. Kenneth Doe III (Kenny)', +# '김민준씨 (Jimmy)' -- which is the reading this entry exists to +# protect. The rest are trailing maiden names and junk +# ('Bridge (1.4)', 'John Jones (Google Docs)'). +# +# So the trailing position is where protection and over-reach arrive +# TOGETHER: a nickname and a credential wear the identical shape +# there, and the delimiter alone cannot separate them. The promise is +# kept for medial pairs only, and a leading or trailing ASCII pair -- +# '"Rick" Edmonds' reads as protected and is not, 'Jenny Baker +# (Johnson)' likewise -- is still unguarded prose. (The corpus also +# holds 'Senator "Rick" Edmonds', which IS medial and IS captured; +# the two are easy to confuse.) Whether the `fields` narrowing already +# makes the wider shape safe is an open question, unmeasured here; +# narrowing a rule is the other way in. +# +# By role: ASCII parens mark nicknames, maiden names, suffixes and +# credentials alike, and no regex tells them apart. The shape reaches +# 'Jenny (Johnson) Baker' and 'Lon (Jr.) Williams', whose parens are a +# maiden name and a suffix -- both areas under active development +# (#335, suffix in parens). `fields` names the two roles a nickname +# regression moves a token BETWEEN, so those names stay classifiable +# on the five it does not name. It does not leave them wholly alone: +# 'Lon (Jr.) Williams' parses its suffix into `middle` today, so a +# middle-only diff on it is silenced. That is the price of protecting +# the reading at all, since a nickname regression is exactly a +# middle/nickname move. +# +# feat(#273)'s own rule needs no shape narrowing of this kind -- a +# typographic delimiter is not a credential marker -- which is why its +# regex is a bare character class. It does carry `fields`, for the +# separate reason recorded there. +name_regex = "[\\w.]\\s+[(\"'][^)\"']+[)\"']\\s+\\w" +fields = ["nickname", "middle"] +examples = ["John \"Jack\" Kennedy", "John (Jack) Kennedy"]