diff --git a/src/mimem/lint/rules.py b/src/mimem/lint/rules.py index ccd17ae..a41a212 100644 --- a/src/mimem/lint/rules.py +++ b/src/mimem/lint/rules.py @@ -185,6 +185,48 @@ class AcronymsAreExpanded(LintRule): CANDIDATE = re.compile(r"\b[A-Z]{2,6}\b(?=[.,;:]?\s)") + #: Capitals either side, with nothing but capitals and punctuation between. A word inside a + #: run like this is capitalised by typography, not because it is an acronym: "RESULTS AND + #: DISCUSSION" is a heading and "Downloaded by UNIV OF TEXAS AT AUSTIN" is a stamp, and + #: between them they accounted for nine warnings about the words "and", "of", "at", "univ", + #: "Texas", "Austin", "review" and "papers". + #: + #: Two neighbours, because a pair of capitals is an acronym beside a name often enough -- + #: "XPS SEM" -- and a run of three is a phrase set in capitals. + IN_CAPS_RUN = 2 + + #: A spoken number after a run of element symbols. ``CH two``, ``SO four``, ``PO four``, + #: ``PF three``, ``WO three``, ``YH three``, ``HNO three`` are formulas whose subscript the + #: number verbalizer has already said, and expanding them is not a thing anyone can do. + #: + #: The number is what makes this safe to act on. ``CV`` and ``PC`` are also element runs -- + #: carbon-vanadium, phosphorus-carbon -- and they are real acronyms, cyclic voltammetry and + #: propylene carbonate; neither is ever followed by a spoken subscript. + _SPOKEN_NUMBER = re.compile( + r"\s+(?:zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)\b", + re.IGNORECASE, + ) + + def _inside_a_caps_run(self, text: str, start: int, end: int) -> bool: + """Is this token one word of a phrase set in capitals?""" + before = text[max(0, start - 60) : start] + after = text[end : end + 60] + left = re.findall(r"[^\W\d_]+", before)[-self.IN_CAPS_RUN :] + right = re.findall(r"[^\W\d_]+", after)[: self.IN_CAPS_RUN] + neighbours = left + right + # At least two letters each: a lone capital beside a token is a variable, not a word + # of a phrase. "R equals ID IG" would otherwise suppress ID because R is capital. + caps = [w for w in neighbours if w.isupper() and len(w) > 1] + return len(caps) >= self.IN_CAPS_RUN + + def _is_a_formula(self, acronym: str, text: str, end: int) -> bool: + """Is this a chemical formula whose subscript has already been spoken?""" + from mimem.verbalize.formulas import element_groups + + if not element_groups(acronym): + return False + return bool(self._SPOKEN_NUMBER.match(text, end)) + #: Units and chemistry that are read as letters and have no expansion to give: "CO two", #: "DC", "pH". Listing them is a judgement call; getting one wrong costs a warning. NOT_ACRONYMS = frozenset({"CO", "DC", "AC", "PH", "UV", "IR", "OK", "II", "III", "IV", "VI"}) @@ -196,14 +238,25 @@ def check(self, text: str) -> list[Violation]: continue if self._expanded(text, acronym): continue - first = re.search(r"\b" + re.escape(acronym) + r"\b", text) - if first is None: # unreachable; the acronym came from this text + # The first occurrence that is a *use*, not the first occurrence. An acronym whose + # first appearance is inside an all-caps heading is still unexpanded everywhere + # else, and skipping it on that evidence hid real ones. + use = next( + ( + m + for m in re.finditer(r"\b" + re.escape(acronym) + r"\b", text) + if not self._inside_a_caps_run(text, m.start(), m.end()) + and not self._is_a_formula(acronym, text, m.end()) + ), + None, + ) + if use is None: continue out.append( self._violation( text, - first.start(), - first.end(), + use.start(), + use.end(), f"{acronym!r} is never expanded; the listener has to guess", ) ) diff --git a/src/mimem/lint/script_rules.py b/src/mimem/lint/script_rules.py index 0de18f6..de2bbbd 100644 --- a/src/mimem/lint/script_rules.py +++ b/src/mimem/lint/script_rules.py @@ -18,10 +18,11 @@ from itertools import pairwise from mimem.config import Profile -from mimem.ir import TEACHING_TYPES, BeatType, ExposureForm, Script +from mimem.ir import TEACHING_TYPES, Beat, BeatType, ExposureForm, Script from mimem.lint.rules import Severity, Violation from mimem.plan.beats import OWNERSHIP_MARKERS from mimem.plan.exposure import exposure_log, spaced +from mimem.verbalize.cohesion import unresolved_opening from mimem.verify import check as check_grounding #: A shared run of this many words is verbatim repetition, not paraphrase (rule REP-01). @@ -596,48 +597,41 @@ class AnaphoraResolvesAcrossBeats(ScriptRule): description = "no beat opens with a pronoun whose antecedent is in another beat" severity = Severity.WARNING - #: Bare pronouns and demonstratives. A demonstrative followed by a noun ("this crust") is - #: resolved by the noun and is not matched -- see ``_BARE`` below. - _PRONOUNS = r"it|this|that|these|those|they|them|such" + @staticmethod + def _with_predecessor(script: Script) -> list[tuple[Beat, Beat | None]]: + """Every beat with the beat spoken immediately before it, ``None`` at a boundary. - #: "It is worth noting", "it turns out", "there is no" -- the subject is grammatical filler - #: and points at nothing, so there is nothing for the listener to have lost. - EXPLETIVE = re.compile( - r"^it\s+(?:is|was|turns\s+out|follows|remains|seems|appears|takes|helps|matters)\b", - re.IGNORECASE, - ) - - #: A demonstrative counts as unresolved only when nothing follows it that could be the - #: referent: "this is why" (bare) fails, "this repair" (determiner) passes. - BARE = re.compile( - r"^(?:" + _PRONOUNS + r")\s+(?:is|are|was|were|means|meant|gives|gave|shows|showed|gets|" - r"got|gets|makes|made|gains|has|have|had|gets|gives|does|do|did|can|could|will|would|" - r"gets|happens|explains|gets|becomes|became|comes|came|goes|went|stays|stayed|leaves|" - r"left|costs|cost|matters|mattered|then|also|too|in|on|at|by|for|with|and|but|so)\b", - re.IGNORECASE, - ) - - #: These two never resolve out loud whatever follows them, so they are matched on their own. - ALWAYS = re.compile(r"^the\s+(?:former|latter|above|aforementioned)\b", re.IGNORECASE) + A segment boundary counts as a boundary; so does the start of the opening or the review + block. What is *inside* a segment is continuous speech. + """ + runs: list[list[Beat]] = [list(script.opening)] + runs += [list(seg.beats) for sec in script.sections for seg in sec.segments] + runs.append(list(script.review)) + return [(beat, run[i - 1] if i else None) for run in runs for i, beat in enumerate(run)] def check(self, script: Script) -> list[Violation]: out: list[Violation] = [] - for beat in script.beats(): - opening = beat.text.strip() - if not opening or beat.generated: + for beat, previous in self._with_predecessor(script): + word = unresolved_opening(beat.text) + if word is None or beat.generated: continue - if self.EXPLETIVE.match(opening): + # **Only when something stands between the pronoun and its antecedent.** Quoting two + # consecutive sentences of a paper in two consecutive beats is not a defect: the + # listener heard the referent three seconds ago, and "...will be discussed in + # further detail. It can be difficult to distinguish..." is how the paper reads. + # + # What breaks it is a boundary. A segment opens, or a transition or a prompt lands in + # front of the pronoun, and the referent is now two beats and a pause away. Four of + # the nine warnings on the corpus were the harmless case. + if previous is not None and not previous.generated: continue - match = self.ALWAYS.match(opening) or self.BARE.match(opening) - if match: - out.append( - self._violation( - f"beat opens with {match.group(0).split()[0].lower()!r}; " - "say the referent instead", - beat.text, - beat.id, - ) + out.append( + self._violation( + f"beat opens with {word!r}; say the referent instead", + beat.text, + beat.id, ) + ) return out diff --git a/src/mimem/plan/planner.py b/src/mimem/plan/planner.py index a206296..63821aa 100644 --- a/src/mimem/plan/planner.py +++ b/src/mimem/plan/planner.py @@ -58,6 +58,7 @@ ) from mimem.render.narrate import uses_superscript_citations from mimem.triage.rules import retained +from mimem.verbalize.cohesion import unresolved_opening #: Prequestions: two to four per document (rule PRQ-01), whatever the profile asks for. PREQUESTION_BOUNDS = (2, 4) @@ -580,6 +581,16 @@ def flush() -> None: or (seconds >= bounds.target_min and seconds + beat.total_seconds > bounds.target_max) ) over_terms = bool(current) and len(new_terms | introduced) > limit + # A boundary in front of a sentence that opens on a bare pronoun is what turns a quoted + # sentence into a broken one (rule SENT-02): "Still on synthesis. This is apparent in + # the superior cycling..." puts a transition and a pause between the pronoun and the + # thing it points at. The break waits one beat, as long as the clock allows. + if ( + (over_time or over_terms) + and unresolved_opening(beat.text) is not None + and seconds + beat.total_seconds <= bounds.hard_max + ): + over_time = over_terms = False if over_time or over_terms: flush() introduced = {c for c in taught if c not in ctx.seen_concepts} @@ -987,6 +998,13 @@ def _cut_point(segment: Segment, target_max: float) -> int | None: break if beat.type is BeatType.PROMPT or following.type is BeatType.ANSWER: continue # rule RET-01: a prompt and its answer are one unit + if unresolved_opening(following.text) is not None: + # Rule SENT-02: the new segment would open on a bare pronoun, with a transition and + # a pause now standing between it and the thing it points at. Every one of the + # corpus's remaining SENT-02 warnings was a cut made here -- "Still on synthesis. + # This is apparent in the superior cycling of the cell..." -- and an earlier boundary + # is always available, because the same loop has already passed one. + continue if running <= target_max: best = i + 1 return best diff --git a/src/mimem/verbalize/cohesion.py b/src/mimem/verbalize/cohesion.py new file mode 100644 index 0000000..9d42b62 --- /dev/null +++ b/src/mimem/verbalize/cohesion.py @@ -0,0 +1,56 @@ +"""Sentences that only work when you have just heard the one before (rule SENT-02). + +"This is apparent in the superior cycling of the cell" is a perfectly good sentence in a paper, +where the reader's eye can go back one line. Spoken, it depends entirely on what was said +immediately before it -- so whether it is a defect is not a property of the sentence at all, but +of what the programme puts in front of it. + +That is why this lives here rather than inside the lint rule: the planner needs the same +judgement when it chooses where to cut a segment, and a boundary placed in front of one of these +is what turns a quoted sentence into a broken one. +""" + +from __future__ import annotations + +import re + +#: Bare pronouns and demonstratives. A demonstrative followed by a noun ("this crust") is +#: resolved by the noun and is not matched -- see :data:`BARE`. +_PRONOUNS = r"it|this|that|these|those|they|them|such" + +#: "It is worth noting", "it turns out", "there is no" -- the subject is grammatical filler and +#: points at nothing, so there is nothing for the listener to have lost. +EXPLETIVE = re.compile( + r"^it\s+(?:is|was|turns\s+out|follows|remains|seems|appears|takes|helps|matters)\b", + re.IGNORECASE, +) + +#: A demonstrative counts as unresolved only when nothing follows it that could be the referent: +#: "this is why" (bare) fails, "this repair" (determiner) passes. +BARE = re.compile( + r"^(?:" + _PRONOUNS + r")\s+(?:is|are|was|were|means|meant|gives|gave|shows|showed|gets|" + r"got|makes|made|gains|has|have|had|does|do|did|can|could|will|would|" + r"happens|explains|becomes|became|comes|came|goes|went|stays|stayed|leaves|" + r"left|costs|cost|matters|mattered|then|also|too|in|on|at|by|for|with|and|but|so)\b", + re.IGNORECASE, +) + +#: These two never resolve out loud whatever follows them, so they are matched on their own. +ALWAYS = re.compile(r"^the\s+(?:former|latter|above|aforementioned)\b", re.IGNORECASE) + + +def unresolved_opening(text: str) -> str | None: + """The word this text leans on, or ``None`` if it stands on its own. + + >>> unresolved_opening("This is apparent in the superior cycling") + 'this' + >>> unresolved_opening("This crust keeps growing") is None + True + >>> unresolved_opening("It is worth noting that the cell failed") is None + True + """ + opening = text.strip() + if not opening or EXPLETIVE.match(opening): + return None + match = ALWAYS.match(opening) or BARE.match(opening) + return match.group(0).split()[0].lower() if match else None diff --git a/tests/unit/test_artefact_lint.py b/tests/unit/test_artefact_lint.py index c693ace..96f94c9 100644 --- a/tests/unit/test_artefact_lint.py +++ b/tests/unit/test_artefact_lint.py @@ -197,3 +197,72 @@ def test_acknowledge_needs_its_object(sentence: str, fires: bool) -> None: re.search(pattern, sentence, re.IGNORECASE) for pattern, _ in DropListRespected.FURNITURE ) assert matched is fires + + +# -- what is not an unexpanded acronym ----------------------------------------------------------- +# +# Of 158 SYM-02 warnings across the stress corpus, 126 were distinct tokens and the majority were +# neither acronyms nor unexpanded. Two classes accounted for most of it. + + +def _flagged(text: str) -> set[str]: + from mimem.lint.rules import AcronymsAreExpanded + + return {v.message.split("'")[1] for v in AcronymsAreExpanded().check(text)} + + +@pytest.mark.parametrize( + "text", + [ + # A word capitalised by typography, not because it is an acronym. The first is a page + # stamp and the second a heading; between them they reported "UNIV", "OF", "TEXAS", "AT", + # "AUSTIN", "AND", "REVIEW" and "PAPERS" as jargon the listener has to guess. + "Downloaded by UNIV OF TEXAS AT AUSTIN on August 27 2015 and the rest followed. ", + "the minimum signal values. RESULTS AND DISCUSSION. We then measured the cells. ", + "This is JES COLLECTION OF INVITED BATTERY REVIEW PAPERS and the review begins here. ", + ], +) +def test_a_word_inside_a_phrase_set_in_capitals(text: str) -> None: + assert _flagged(text) == set() + + +@pytest.mark.parametrize( + ("text", "formula"), + [ + # A chemical formula whose subscript the number verbalizer has already said. Expanding + # one is not a thing anyone can do. + ("aqueous solutions of H two SO four and acid were used here. ", "SO"), + ("the transition metal oxides WO three were prepared by heating. ", "WO"), + ("semiconductor yttrium trihydride, YH three, was measured at once. ", "YH"), + ("peels treated with H three PO four for one hour in the bath. ", "PO"), + ("signals characteristic of CH two groups were seen in the spectrum. ", "CH"), + ], +) +def test_a_formula_whose_subscript_has_already_been_spoken(text: str, formula: str) -> None: + assert formula not in _flagged(text) + + +@pytest.mark.parametrize( + ("text", "acronym"), + [ + # The number is what makes the formula test safe to act on. These are element runs too -- + # carbon-vanadium, phosphorus-carbon -- and they are real acronyms that are never + # followed by a spoken subscript. + ("measured by CV at a scan rate of five millivolts per second here. ", "CV"), + ("the PC electrolyte was degassed before use in every cell we built. ", "PC"), + # ...and the ordinary case the rule exists for. + ("images were taken with SEM and the surface was clean throughout. ", "SEM"), + ("the SEI layer grew thicker on each cycle of the cell under test. ", "SEI"), + ], +) +def test_an_acronym_the_source_never_expands_is_still_reported(text: str, acronym: str) -> None: + assert acronym in _flagged(text) + + +def test_an_acronym_is_reported_on_its_first_real_use_not_its_first_appearance() -> None: + """Skipping it because a heading came first hid ``ATR`` and ``SBR``. + + Both are genuinely never expanded, and both appear in running prose further down. + """ + text = "ATR FTIR SPECTROSCOPY OF ELECTRODES. The spectra were taken by ATR on a sample. " + assert "ATR" in _flagged(text) diff --git a/tests/unit/test_cohesion.py b/tests/unit/test_cohesion.py new file mode 100644 index 0000000..d95e2ba --- /dev/null +++ b/tests/unit/test_cohesion.py @@ -0,0 +1,101 @@ +"""Sentences that lean on the one before them, and the boundaries that break them. + +Rule SENT-02 was reporting nine warnings across the corpus and four of them were harmless: two +consecutive sentences of a paper, quoted in two consecutive beats, where the listener heard the +referent three seconds ago. The other five were real, and all five were boundaries the planner +had chosen -- a transition and a pause inserted between a pronoun and the thing it points at. +""" + +from __future__ import annotations + +import pytest + +from mimem.config import Profile +from mimem.ir import BeatType, Script +from mimem.lint.script_rules import AnaphoraResolvesAcrossBeats +from mimem.verbalize.cohesion import unresolved_opening + + +@pytest.mark.parametrize( + ("text", "word"), + [ + ("This is apparent in the superior cycling of the cell.", "this"), + ("They were directly mounted on the tightly closed ATR unit.", "they"), + ("This would decrease the significance of side reactions.", "this"), + ("The latter was measured at room temperature.", "the"), + ("Such was the case for every sample.", "such"), + ], +) +def test_a_sentence_that_points_at_the_one_before_it(text: str, word: str) -> None: + assert unresolved_opening(text) == word + + +@pytest.mark.parametrize( + "text", + [ + # A demonstrative resolved by the noun that follows it. + "This crust keeps growing at every cycle.", + "These electrodes were cycled a hundred times.", + # An expletive subject points at nothing, so nothing has been lost. + "It is worth noting that the cell failed early.", + "It turns out the binder was the problem.", + # And an ordinary opening. + "The clamping stress differs between the two resonators.", + ], +) +def test_a_sentence_that_stands_on_its_own(text: str) -> None: + assert unresolved_opening(text) is None + + +def _script(*runs: list[str]) -> Script: + """A script whose segments hold the given beat texts, none of them generated.""" + from mimem.ir import Beat, Section, Segment, SourceMeta + + sections = [ + Section( + id="s1", + title="Results", + order=1, + segments=[ + Segment( + id=f"g{i}", + section_id="s1", + beats=[ + Beat( + id=f"b{i}_{j}", + type=BeatType.EXPOSITION, + text=text, + generated=False, + ) + for j, text in enumerate(run) + ], + ) + for i, run in enumerate(runs) + ], + ) + ] + return Script(doc_id="d", source=SourceMeta(format="pdf"), profile="study", sections=sections) + + +def test_two_quoted_sentences_in_a_row_are_not_a_defect() -> None: + """The referent was spoken three seconds ago, and this is how the paper reads.""" + rule = AnaphoraResolvesAcrossBeats(Profile(name="study")) + script = _script( + [ + "Peaks were observed in one experiment and will be discussed in further detail.", + "This makes it difficult to determine the origin of these species.", + ] + ) + assert rule.check(script) == [] + + +def test_a_boundary_in_front_of_the_pronoun_is_a_defect() -> None: + """Same two sentences, a segment boundary between them.""" + rule = AnaphoraResolvesAcrossBeats(Profile(name="study")) + script = _script( + ["Peaks were observed in one experiment and will be discussed in further detail."], + ["This makes it difficult to determine the origin of these species."], + ) + violations = rule.check(script) + assert len(violations) == 1 + assert "'this'" in violations[0].message diff --git a/tests/unit/test_rule_triggers.py b/tests/unit/test_rule_triggers.py index 388094a..587f1bd 100644 --- a/tests/unit/test_rule_triggers.py +++ b/tests/unit/test_rule_triggers.py @@ -344,16 +344,19 @@ def section(index: str, beats: list[Beat]) -> Section: # A chunk that stops mid-sentence. "TTS-04": lambda: _script(segments=[[_beat("e1", text="The residual drift is dominated by")]]), # A beat opening on a pronoun with no referent in it. + # The pronoun has to open a *segment*. Quoting two consecutive sentences of a paper in two + # consecutive beats is not the defect -- the listener heard the referent three seconds ago. + # What breaks it is the boundary: a transition and a pause now stand in between. "SENT-02": lambda: _script( segments=[ + [_beat("e1", text="The clamping stress differs between the two resonators.")], [ - _beat("e1", text="The clamping stress differs between the two resonators."), _beat( "e2", text="This is what sets the floor on paired compensation.", generated=False, - ), - ] + ) + ], ] ), # A table read out before the listener is told it is a table.