A clause is not a concept

src/mimem/concepts/extract.py

@@ -215,7 +215,15 @@
 #: real subjects -- it repeats, and repetition is all the window can see.
 CLAUSE_LIKE = re.compile(
     r"\b(?:is|are|was|were|be|been|being|has|have|had|do|does|did|will|would|can|could|"
-    r"should|may|might|must)\b",
+    r"should|may|might|must|"
+    # The ones that carry the meaning rather than the tense. Without them "scan shows the xrd"
+    # is a term: it survives the boundary check, because the verb is in the middle.
+    r"shows|show|showed|reveals|reveal|revealed|indicates|indicate|indicated|"
+    r"suggests|suggest|suggested|contains|contain|contained|exhibits|exhibit|exhibited|"
+    r"becomes|become|became|appears|appear|appeared|remains|remain|remained|"
+    # "lead" is not here, and neither is "leads": in a battery corpus it is the metal, and a
+    # rule that loses "lead acid battery" to catch "leads to capacity fade" is a bad trade.
+    r"gives|give|gave|makes|make|made)\b",
     re.IGNORECASE,
 )
 
@@ -268,6 +276,51 @@
         "proposed",
         "developed",
         "conducted",
+        # -- the present tense, which the list did not have at all ---------------------------
+        #
+        # Every entry above is a past participle, so a phrase that opens or closes on a finite
+        # verb went straight through: "shows the XRD pattern", "figure shows", "scan shows the
+        # xrd" were three of one paper's twenty-seven concepts, and the programme asked "what did
+        # they report for figure shows?"
+        "shows",
+        "show",
+        "reveals",
+        "reveal",
+        "indicates",
+        "indicate",
+        "suggests",
+        "suggest",
+        "becomes",
+        "become",
+        "contains",
+        "contain",
+        "exhibits",
+        "exhibit",
+        "see",
+        # -- participles the corpus supplied, and their -ing forms ----------------------------
+        "cycled",
+        "cycling",
+        "prepared",
+        "preparing",
+        "discharged",
+        "discharging",
+        "charged",
+        "charging",
+        "stopped",
+        "extracted",
+        "containing",
+        "showing",
+        "associated",
+        "characteristic",
+        "consistent",
+        "attributed",
+        "assigned",
+        "related",
+        "corresponding",
+        # -- and a preposition the stop list missed --------------------------------------------
+        "against",
+        "above",
+        "below",
     ]
 )
 

tests/unit/test_concepts.py

@@ -280,3 +280,63 @@ def test_norms_status_is_honest_about_its_source() -> None:
     assert reported.source in {"brysbaert", "morphology"}
     if reported.source == "morphology":
         assert reported.entries == 0
+
+
+# -- phrases that are clauses, which the verb lists could not see --------------------------------
+#
+# 38 of the stress corpus's 482 concepts were clause fragments, and the programme asked questions
+# about them: "What did they report for figure shows?" Both lists were written in the past tense.
+
+
+def _is_a_term(phrase: str) -> bool:
+    from mimem.concepts.extract import CLAUSE_LIKE, _is_phrase_like
+
+    return _is_phrase_like(phrase.split()) and not CLAUSE_LIKE.search(phrase)
+
+
+@pytest.mark.parametrize(
+    "fragment",
+    [
+        # A finite verb at the boundary. PHRASE_BOUNDARY_VERBS had only past participles.
+        "figure shows",
+        "shows the XRD pattern",
+        "see Figure",
+        "see Table",
+        "becomes amorphous",
+        "containing species",
+        "consistent with the presence",
+        "peaks characteristic",
+        "capacity associated",
+        "electrodes extracted",
+        "against sodium metal counter",
+        "discharged to mv",
+        "above mv",
+        # A finite verb in the *middle*, which the boundary check cannot reach. This is what
+        # CLAUSE_LIKE is for, and it had only the auxiliaries.
+        "scan shows the xrd",
+        "study are open-sourced",
+    ],
+)
+def test_a_clause_is_not_a_concept(fragment: str) -> None:
+    assert not _is_a_term(fragment)
+
+
+@pytest.mark.parametrize(
+    "term",
+    [
+        # Ordinary terms, which must survive both lists.
+        "crystalline silicon",
+        "irreversible capacity",
+        "solid electrolyte interphase",
+        "differential capacity",
+        "state of charge",
+        "battery degradation",
+        "oxygen-containing yttrium hydride",
+        # "lead" is the metal here. A rule that loses this to catch "leads to capacity fade" is a
+        # bad trade, so the lead family is deliberately absent from CLAUSE_LIKE.
+        "lead acid battery",
+        "lithium lead alloy",
+    ],
+)
+def test_a_term_survives_the_verb_lists(term: str) -> None:
+    assert _is_a_term(term)