diff --git a/src/mimem/concepts/extract.py b/src/mimem/concepts/extract.py index aba9874..f153fc7 100644 --- a/src/mimem/concepts/extract.py +++ b/src/mimem/concepts/extract.py @@ -215,7 +215,15 @@ #: real subjects -- it repeats, and repetition is all the window can see. CLAUSE_LIKE = re.compile( r"\b(?:is|are|was|were|be|been|being|has|have|had|do|does|did|will|would|can|could|" - r"should|may|might|must)\b", + r"should|may|might|must|" + # The ones that carry the meaning rather than the tense. Without them "scan shows the xrd" + # is a term: it survives the boundary check, because the verb is in the middle. + r"shows|show|showed|reveals|reveal|revealed|indicates|indicate|indicated|" + r"suggests|suggest|suggested|contains|contain|contained|exhibits|exhibit|exhibited|" + r"becomes|become|became|appears|appear|appeared|remains|remain|remained|" + # "lead" is not here, and neither is "leads": in a battery corpus it is the metal, and a + # rule that loses "lead acid battery" to catch "leads to capacity fade" is a bad trade. + r"gives|give|gave|makes|make|made)\b", re.IGNORECASE, ) @@ -268,6 +276,51 @@ "proposed", "developed", "conducted", + # -- the present tense, which the list did not have at all --------------------------- + # + # Every entry above is a past participle, so a phrase that opens or closes on a finite + # verb went straight through: "shows the XRD pattern", "figure shows", "scan shows the + # xrd" were three of one paper's twenty-seven concepts, and the programme asked "what did + # they report for figure shows?" + "shows", + "show", + "reveals", + "reveal", + "indicates", + "indicate", + "suggests", + "suggest", + "becomes", + "become", + "contains", + "contain", + "exhibits", + "exhibit", + "see", + # -- participles the corpus supplied, and their -ing forms ---------------------------- + "cycled", + "cycling", + "prepared", + "preparing", + "discharged", + "discharging", + "charged", + "charging", + "stopped", + "extracted", + "containing", + "showing", + "associated", + "characteristic", + "consistent", + "attributed", + "assigned", + "related", + "corresponding", + # -- and a preposition the stop list missed -------------------------------------------- + "against", + "above", + "below", ] ) diff --git a/tests/unit/test_concepts.py b/tests/unit/test_concepts.py index bbe260d..324baae 100644 --- a/tests/unit/test_concepts.py +++ b/tests/unit/test_concepts.py @@ -280,3 +280,63 @@ def test_norms_status_is_honest_about_its_source() -> None: assert reported.source in {"brysbaert", "morphology"} if reported.source == "morphology": assert reported.entries == 0 + + +# -- phrases that are clauses, which the verb lists could not see -------------------------------- +# +# 38 of the stress corpus's 482 concepts were clause fragments, and the programme asked questions +# about them: "What did they report for figure shows?" Both lists were written in the past tense. + + +def _is_a_term(phrase: str) -> bool: + from mimem.concepts.extract import CLAUSE_LIKE, _is_phrase_like + + return _is_phrase_like(phrase.split()) and not CLAUSE_LIKE.search(phrase) + + +@pytest.mark.parametrize( + "fragment", + [ + # A finite verb at the boundary. PHRASE_BOUNDARY_VERBS had only past participles. + "figure shows", + "shows the XRD pattern", + "see Figure", + "see Table", + "becomes amorphous", + "containing species", + "consistent with the presence", + "peaks characteristic", + "capacity associated", + "electrodes extracted", + "against sodium metal counter", + "discharged to mv", + "above mv", + # A finite verb in the *middle*, which the boundary check cannot reach. This is what + # CLAUSE_LIKE is for, and it had only the auxiliaries. + "scan shows the xrd", + "study are open-sourced", + ], +) +def test_a_clause_is_not_a_concept(fragment: str) -> None: + assert not _is_a_term(fragment) + + +@pytest.mark.parametrize( + "term", + [ + # Ordinary terms, which must survive both lists. + "crystalline silicon", + "irreversible capacity", + "solid electrolyte interphase", + "differential capacity", + "state of charge", + "battery degradation", + "oxygen-containing yttrium hydride", + # "lead" is the metal here. A rule that loses this to catch "leads to capacity fade" is a + # bad trade, so the lead family is deliberately absent from CLAUSE_LIKE. + "lead acid battery", + "lithium lead alloy", + ], +) +def test_a_term_survives_the_verb_lists(term: str) -> None: + assert _is_a_term(term)