Stop two rules reporting things that are not defects

src/mimem/lint/rules.py

@@ -185,6 +185,48 @@ class AcronymsAreExpanded(LintRule):
 
     CANDIDATE = re.compile(r"\b[A-Z]{2,6}\b(?=[.,;:]?\s)")
 
+    #: Capitals either side, with nothing but capitals and punctuation between. A word inside a
+    #: run like this is capitalised by typography, not because it is an acronym: "RESULTS AND
+    #: DISCUSSION" is a heading and "Downloaded by UNIV OF TEXAS AT AUSTIN" is a stamp, and
+    #: between them they accounted for nine warnings about the words "and", "of", "at", "univ",
+    #: "Texas", "Austin", "review" and "papers".
+    #:
+    #: Two neighbours, because a pair of capitals is an acronym beside a name often enough --
+    #: "XPS SEM" -- and a run of three is a phrase set in capitals.
+    IN_CAPS_RUN = 2
+
+    #: A spoken number after a run of element symbols. ``CH two``, ``SO four``, ``PO four``,
+    #: ``PF three``, ``WO three``, ``YH three``, ``HNO three`` are formulas whose subscript the
+    #: number verbalizer has already said, and expanding them is not a thing anyone can do.
+    #:
+    #: The number is what makes this safe to act on. ``CV`` and ``PC`` are also element runs --
+    #: carbon-vanadium, phosphorus-carbon -- and they are real acronyms, cyclic voltammetry and
+    #: propylene carbonate; neither is ever followed by a spoken subscript.
+    _SPOKEN_NUMBER = re.compile(
+        r"\s+(?:zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)\b",
+        re.IGNORECASE,
+    )
+
+    def _inside_a_caps_run(self, text: str, start: int, end: int) -> bool:
+        """Is this token one word of a phrase set in capitals?"""
+        before = text[max(0, start - 60) : start]
+        after = text[end : end + 60]
+        left = re.findall(r"[^\W\d_]+", before)[-self.IN_CAPS_RUN :]
+        right = re.findall(r"[^\W\d_]+", after)[: self.IN_CAPS_RUN]
+        neighbours = left + right
+        # At least two letters each: a lone capital beside a token is a variable, not a word
+        # of a phrase. "R equals ID IG" would otherwise suppress ID because R is capital.
+        caps = [w for w in neighbours if w.isupper() and len(w) > 1]
+        return len(caps) >= self.IN_CAPS_RUN
+
+    def _is_a_formula(self, acronym: str, text: str, end: int) -> bool:
+        """Is this a chemical formula whose subscript has already been spoken?"""
+        from mimem.verbalize.formulas import element_groups
+
+        if not element_groups(acronym):
+            return False
+        return bool(self._SPOKEN_NUMBER.match(text, end))
+
     #: Units and chemistry that are read as letters and have no expansion to give: "CO two",
     #: "DC", "pH". Listing them is a judgement call; getting one wrong costs a warning.
     NOT_ACRONYMS = frozenset({"CO", "DC", "AC", "PH", "UV", "IR", "OK", "II", "III", "IV", "VI"})
@@ -196,14 +238,25 @@ def check(self, text: str) -> list[Violation]:
                 continue
             if self._expanded(text, acronym):
                 continue
-            first = re.search(r"\b" + re.escape(acronym) + r"\b", text)
-            if first is None:  # unreachable; the acronym came from this text
+            # The first occurrence that is a *use*, not the first occurrence. An acronym whose
+            # first appearance is inside an all-caps heading is still unexpanded everywhere
+            # else, and skipping it on that evidence hid real ones.
+            use = next(
+                (
+                    m
+                    for m in re.finditer(r"\b" + re.escape(acronym) + r"\b", text)
+                    if not self._inside_a_caps_run(text, m.start(), m.end())
+                    and not self._is_a_formula(acronym, text, m.end())
+                ),
+                None,
+            )
+            if use is None:
                 continue
             out.append(
                 self._violation(
                     text,
-                    first.start(),
-                    first.end(),
+                    use.start(),
+                    use.end(),
                     f"{acronym!r} is never expanded; the listener has to guess",
                 )
             )

src/mimem/lint/script_rules.py

@@ -18,10 +18,11 @@
 from itertools import pairwise
 
 from mimem.config import Profile
-from mimem.ir import TEACHING_TYPES, BeatType, ExposureForm, Script
+from mimem.ir import TEACHING_TYPES, Beat, BeatType, ExposureForm, Script
 from mimem.lint.rules import Severity, Violation
 from mimem.plan.beats import OWNERSHIP_MARKERS
 from mimem.plan.exposure import exposure_log, spaced
+from mimem.verbalize.cohesion import unresolved_opening
 from mimem.verify import check as check_grounding
 
 #: A shared run of this many words is verbatim repetition, not paraphrase (rule REP-01).
@@ -596,48 +597,41 @@ class AnaphoraResolvesAcrossBeats(ScriptRule):
     description = "no beat opens with a pronoun whose antecedent is in another beat"
     severity = Severity.WARNING
 
-    #: Bare pronouns and demonstratives. A demonstrative followed by a noun ("this crust") is
-    #: resolved by the noun and is not matched -- see ``_BARE`` below.
-    _PRONOUNS = r"it|this|that|these|those|they|them|such"
+    @staticmethod
+    def _with_predecessor(script: Script) -> list[tuple[Beat, Beat | None]]:
+        """Every beat with the beat spoken immediately before it, ``None`` at a boundary.
 
-    #: "It is worth noting", "it turns out", "there is no" -- the subject is grammatical filler
-    #: and points at nothing, so there is nothing for the listener to have lost.
-    EXPLETIVE = re.compile(
-        r"^it\s+(?:is|was|turns\s+out|follows|remains|seems|appears|takes|helps|matters)\b",
-        re.IGNORECASE,
-    )
-
-    #: A demonstrative counts as unresolved only when nothing follows it that could be the
-    #: referent: "this is why" (bare) fails, "this repair" (determiner) passes.
-    BARE = re.compile(
-        r"^(?:" + _PRONOUNS + r")\s+(?:is|are|was|were|means|meant|gives|gave|shows|showed|gets|"
-        r"got|gets|makes|made|gains|has|have|had|gets|gives|does|do|did|can|could|will|would|"
-        r"gets|happens|explains|gets|becomes|became|comes|came|goes|went|stays|stayed|leaves|"
-        r"left|costs|cost|matters|mattered|then|also|too|in|on|at|by|for|with|and|but|so)\b",
-        re.IGNORECASE,
-    )
-
-    #: These two never resolve out loud whatever follows them, so they are matched on their own.
-    ALWAYS = re.compile(r"^the\s+(?:former|latter|above|aforementioned)\b", re.IGNORECASE)
+        A segment boundary counts as a boundary; so does the start of the opening or the review
+        block. What is *inside* a segment is continuous speech.
+        """
+        runs: list[list[Beat]] = [list(script.opening)]
+        runs += [list(seg.beats) for sec in script.sections for seg in sec.segments]
+        runs.append(list(script.review))
+        return [(beat, run[i - 1] if i else None) for run in runs for i, beat in enumerate(run)]
 
     def check(self, script: Script) -> list[Violation]:
         out: list[Violation] = []
-        for beat in script.beats():
-            opening = beat.text.strip()
-            if not opening or beat.generated:
+        for beat, previous in self._with_predecessor(script):
+            word = unresolved_opening(beat.text)
+            if word is None or beat.generated:
                 continue
-            if self.EXPLETIVE.match(opening):
+            # **Only when something stands between the pronoun and its antecedent.** Quoting two
+            # consecutive sentences of a paper in two consecutive beats is not a defect: the
+            # listener heard the referent three seconds ago, and "...will be discussed in
+            # further detail. It can be difficult to distinguish..." is how the paper reads.
+            #
+            # What breaks it is a boundary. A segment opens, or a transition or a prompt lands in
+            # front of the pronoun, and the referent is now two beats and a pause away. Four of
+            # the nine warnings on the corpus were the harmless case.
+            if previous is not None and not previous.generated:
                 continue
-            match = self.ALWAYS.match(opening) or self.BARE.match(opening)
-            if match:
-                out.append(
-                    self._violation(
-                        f"beat opens with {match.group(0).split()[0].lower()!r}; "
-                        "say the referent instead",
-                        beat.text,
-                        beat.id,
-                    )
+            out.append(
+                self._violation(
+                    f"beat opens with {word!r}; say the referent instead",
+                    beat.text,
+                    beat.id,
                 )
+            )
         return out
 
 

src/mimem/plan/planner.py

@@ -58,6 +58,7 @@
 )
 from mimem.render.narrate import uses_superscript_citations
 from mimem.triage.rules import retained
+from mimem.verbalize.cohesion import unresolved_opening
 
 #: Prequestions: two to four per document (rule PRQ-01), whatever the profile asks for.
 PREQUESTION_BOUNDS = (2, 4)
@@ -580,6 +581,16 @@ def flush() -> None:
             or (seconds >= bounds.target_min and seconds + beat.total_seconds > bounds.target_max)
         )
         over_terms = bool(current) and len(new_terms | introduced) > limit
+        # A boundary in front of a sentence that opens on a bare pronoun is what turns a quoted
+        # sentence into a broken one (rule SENT-02): "Still on synthesis. This is apparent in
+        # the superior cycling..." puts a transition and a pause between the pronoun and the
+        # thing it points at. The break waits one beat, as long as the clock allows.
+        if (
+            (over_time or over_terms)
+            and unresolved_opening(beat.text) is not None
+            and seconds + beat.total_seconds <= bounds.hard_max
+        ):
+            over_time = over_terms = False
         if over_time or over_terms:
             flush()
             introduced = {c for c in taught if c not in ctx.seen_concepts}
@@ -987,6 +998,13 @@ def _cut_point(segment: Segment, target_max: float) -> int | None:
             break
         if beat.type is BeatType.PROMPT or following.type is BeatType.ANSWER:
             continue  # rule RET-01: a prompt and its answer are one unit
+        if unresolved_opening(following.text) is not None:
+            # Rule SENT-02: the new segment would open on a bare pronoun, with a transition and
+            # a pause now standing between it and the thing it points at. Every one of the
+            # corpus's remaining SENT-02 warnings was a cut made here -- "Still on synthesis.
+            # This is apparent in the superior cycling of the cell..." -- and an earlier boundary
+            # is always available, because the same loop has already passed one.
+            continue
         if running <= target_max:
             best = i + 1
     return best

src/mimem/verbalize/cohesion.py

@@ -0,0 +1,56 @@
+"""Sentences that only work when you have just heard the one before (rule SENT-02).
+
+"This is apparent in the superior cycling of the cell" is a perfectly good sentence in a paper,
+where the reader's eye can go back one line. Spoken, it depends entirely on what was said
+immediately before it -- so whether it is a defect is not a property of the sentence at all, but
+of what the programme puts in front of it.
+
+That is why this lives here rather than inside the lint rule: the planner needs the same
+judgement when it chooses where to cut a segment, and a boundary placed in front of one of these
+is what turns a quoted sentence into a broken one.
+"""
+
+from __future__ import annotations
+
+import re
+
+#: Bare pronouns and demonstratives. A demonstrative followed by a noun ("this crust") is
+#: resolved by the noun and is not matched -- see :data:`BARE`.
+_PRONOUNS = r"it|this|that|these|those|they|them|such"
+
+#: "It is worth noting", "it turns out", "there is no" -- the subject is grammatical filler and
+#: points at nothing, so there is nothing for the listener to have lost.
+EXPLETIVE = re.compile(
+    r"^it\s+(?:is|was|turns\s+out|follows|remains|seems|appears|takes|helps|matters)\b",
+    re.IGNORECASE,
+)
+
+#: A demonstrative counts as unresolved only when nothing follows it that could be the referent:
+#: "this is why" (bare) fails, "this repair" (determiner) passes.
+BARE = re.compile(
+    r"^(?:" + _PRONOUNS + r")\s+(?:is|are|was|were|means|meant|gives|gave|shows|showed|gets|"
+    r"got|makes|made|gains|has|have|had|does|do|did|can|could|will|would|"
+    r"happens|explains|becomes|became|comes|came|goes|went|stays|stayed|leaves|"
+    r"left|costs|cost|matters|mattered|then|also|too|in|on|at|by|for|with|and|but|so)\b",
+    re.IGNORECASE,
+)
+
+#: These two never resolve out loud whatever follows them, so they are matched on their own.
+ALWAYS = re.compile(r"^the\s+(?:former|latter|above|aforementioned)\b", re.IGNORECASE)
+
+
+def unresolved_opening(text: str) -> str | None:
+    """The word this text leans on, or ``None`` if it stands on its own.
+
+    >>> unresolved_opening("This is apparent in the superior cycling")
+    'this'
+    >>> unresolved_opening("This crust keeps growing") is None
+    True
+    >>> unresolved_opening("It is worth noting that the cell failed") is None
+    True
+    """
+    opening = text.strip()
+    if not opening or EXPLETIVE.match(opening):
+        return None
+    match = ALWAYS.match(opening) or BARE.match(opening)
+    return match.group(0).split()[0].lower() if match else None

tests/unit/test_artefact_lint.py

@@ -197,3 +197,72 @@ def test_acknowledge_needs_its_object(sentence: str, fires: bool) -> None:
         re.search(pattern, sentence, re.IGNORECASE) for pattern, _ in DropListRespected.FURNITURE
     )
     assert matched is fires
+
+
+# -- what is not an unexpanded acronym -----------------------------------------------------------
+#
+# Of 158 SYM-02 warnings across the stress corpus, 126 were distinct tokens and the majority were
+# neither acronyms nor unexpanded. Two classes accounted for most of it.
+
+
+def _flagged(text: str) -> set[str]:
+    from mimem.lint.rules import AcronymsAreExpanded
+
+    return {v.message.split("'")[1] for v in AcronymsAreExpanded().check(text)}
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        # A word capitalised by typography, not because it is an acronym. The first is a page
+        # stamp and the second a heading; between them they reported "UNIV", "OF", "TEXAS", "AT",
+        # "AUSTIN", "AND", "REVIEW" and "PAPERS" as jargon the listener has to guess.
+        "Downloaded by UNIV OF TEXAS AT AUSTIN on August 27 2015 and the rest followed. ",
+        "the minimum signal values. RESULTS AND DISCUSSION. We then measured the cells. ",
+        "This is JES COLLECTION OF INVITED BATTERY REVIEW PAPERS and the review begins here. ",
+    ],
+)
+def test_a_word_inside_a_phrase_set_in_capitals(text: str) -> None:
+    assert _flagged(text) == set()
+
+
+@pytest.mark.parametrize(
+    ("text", "formula"),
+    [
+        # A chemical formula whose subscript the number verbalizer has already said. Expanding
+        # one is not a thing anyone can do.
+        ("aqueous solutions of H two SO four and acid were used here. ", "SO"),
+        ("the transition metal oxides WO three were prepared by heating. ", "WO"),
+        ("semiconductor yttrium trihydride, YH three, was measured at once. ", "YH"),
+        ("peels treated with H three PO four for one hour in the bath. ", "PO"),
+        ("signals characteristic of CH two groups were seen in the spectrum. ", "CH"),
+    ],
+)
+def test_a_formula_whose_subscript_has_already_been_spoken(text: str, formula: str) -> None:
+    assert formula not in _flagged(text)
+
+
+@pytest.mark.parametrize(
+    ("text", "acronym"),
+    [
+        # The number is what makes the formula test safe to act on. These are element runs too --
+        # carbon-vanadium, phosphorus-carbon -- and they are real acronyms that are never
+        # followed by a spoken subscript.
+        ("measured by CV at a scan rate of five millivolts per second here. ", "CV"),
+        ("the PC electrolyte was degassed before use in every cell we built. ", "PC"),
+        # ...and the ordinary case the rule exists for.
+        ("images were taken with SEM and the surface was clean throughout. ", "SEM"),
+        ("the SEI layer grew thicker on each cycle of the cell under test. ", "SEI"),
+    ],
+)
+def test_an_acronym_the_source_never_expands_is_still_reported(text: str, acronym: str) -> None:
+    assert acronym in _flagged(text)
+
+
+def test_an_acronym_is_reported_on_its_first_real_use_not_its_first_appearance() -> None:
+    """Skipping it because a heading came first hid ``ATR`` and ``SBR``.
+
+    Both are genuinely never expanded, and both appear in running prose further down.
+    """
+    text = "ATR FTIR SPECTROSCOPY OF ELECTRODES. The spectra were taken by ATR on a sample. "
+    assert "ATR" in _flagged(text)