From 0726c8d99fc4b019d4ecfba3003544dc405976b3 Mon Sep 17 00:00:00 2001 From: jepegit Date: Wed, 9 Sep 2026 08:48:08 +0200 Subject: [PATCH] A clause is not a concept Thirty-eight of the stress corpus's 482 concepts were clause fragments, and the programme asked questions about them: "What did they report for figure shows?" One paper had "figure shows", "shows the XRD pattern" and "scan shows the xrd" among its twenty-seven concepts, another had "see Figure" and "see Table". Both guards were written in the past tense. PHRASE_BOUNDARY_VERBS is thirty-five entries and every one of them is a past participle, so a phrase opening or closing on a finite verb went straight through. CLAUSE_LIKE -- the rule whose own docstring says "a finite verb makes a phrase a clause rather than a term" -- had the auxiliaries and nothing else, so a verb in the *middle* was invisible to both: the boundary check cannot see it and CLAUSE_LIKE did not know the word. The lead family is deliberately absent from CLAUSE_LIKE. In a battery corpus "lead" is the metal, and a rule that loses "lead acid battery" to catch "leads to capacity fade" is a bad trade. Clause fragments across the corpus fall from 38 to 8, concepts from 482 to 442, and SEG-03 from 27 to 23 -- a segment reported as introducing five new terms when two of them were the same noun phrase cut in different places was the working-memory rule being broken by arithmetic rather than by content. The eight that remain are reduced relatives: "electrodes cycled with LiPF6", "electrode cycled in EC". They are at least comprehensible as terms, and the judgement that separates them from "cycled electrode surface" is not one a word list can make. What this does not fix is the overlap: forty-three pairs where one concept is a sub-phrase of another, "metal counter" beside "metal counter electrodes" beside "sodium metal counter electrodes". _drop_redundant already decides those on the ratio of their mention counts, and measuring the corpus says the ratio cannot: the truncation "metal counter" sits at 0.75, between "thin film" and "specific surface", which are both real terms. What separates them is that a truncation ends on a modifier rather than a head noun, which is a judgement about English that a regex does not have. Left alone rather than guessed at. Co-Authored-By: Claude Opus 5 --- src/mimem/concepts/extract.py | 55 +++++++++++++++++++++++++++++++- tests/unit/test_concepts.py | 60 +++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/src/mimem/concepts/extract.py b/src/mimem/concepts/extract.py index aba9874..f153fc7 100644 --- a/src/mimem/concepts/extract.py +++ b/src/mimem/concepts/extract.py @@ -215,7 +215,15 @@ #: real subjects -- it repeats, and repetition is all the window can see. CLAUSE_LIKE = re.compile( r"\b(?:is|are|was|were|be|been|being|has|have|had|do|does|did|will|would|can|could|" - r"should|may|might|must)\b", + r"should|may|might|must|" + # The ones that carry the meaning rather than the tense. Without them "scan shows the xrd" + # is a term: it survives the boundary check, because the verb is in the middle. + r"shows|show|showed|reveals|reveal|revealed|indicates|indicate|indicated|" + r"suggests|suggest|suggested|contains|contain|contained|exhibits|exhibit|exhibited|" + r"becomes|become|became|appears|appear|appeared|remains|remain|remained|" + # "lead" is not here, and neither is "leads": in a battery corpus it is the metal, and a + # rule that loses "lead acid battery" to catch "leads to capacity fade" is a bad trade. + r"gives|give|gave|makes|make|made)\b", re.IGNORECASE, ) @@ -268,6 +276,51 @@ "proposed", "developed", "conducted", + # -- the present tense, which the list did not have at all --------------------------- + # + # Every entry above is a past participle, so a phrase that opens or closes on a finite + # verb went straight through: "shows the XRD pattern", "figure shows", "scan shows the + # xrd" were three of one paper's twenty-seven concepts, and the programme asked "what did + # they report for figure shows?" + "shows", + "show", + "reveals", + "reveal", + "indicates", + "indicate", + "suggests", + "suggest", + "becomes", + "become", + "contains", + "contain", + "exhibits", + "exhibit", + "see", + # -- participles the corpus supplied, and their -ing forms ---------------------------- + "cycled", + "cycling", + "prepared", + "preparing", + "discharged", + "discharging", + "charged", + "charging", + "stopped", + "extracted", + "containing", + "showing", + "associated", + "characteristic", + "consistent", + "attributed", + "assigned", + "related", + "corresponding", + # -- and a preposition the stop list missed -------------------------------------------- + "against", + "above", + "below", ] ) diff --git a/tests/unit/test_concepts.py b/tests/unit/test_concepts.py index bbe260d..324baae 100644 --- a/tests/unit/test_concepts.py +++ b/tests/unit/test_concepts.py @@ -280,3 +280,63 @@ def test_norms_status_is_honest_about_its_source() -> None: assert reported.source in {"brysbaert", "morphology"} if reported.source == "morphology": assert reported.entries == 0 + + +# -- phrases that are clauses, which the verb lists could not see -------------------------------- +# +# 38 of the stress corpus's 482 concepts were clause fragments, and the programme asked questions +# about them: "What did they report for figure shows?" Both lists were written in the past tense. + + +def _is_a_term(phrase: str) -> bool: + from mimem.concepts.extract import CLAUSE_LIKE, _is_phrase_like + + return _is_phrase_like(phrase.split()) and not CLAUSE_LIKE.search(phrase) + + +@pytest.mark.parametrize( + "fragment", + [ + # A finite verb at the boundary. PHRASE_BOUNDARY_VERBS had only past participles. + "figure shows", + "shows the XRD pattern", + "see Figure", + "see Table", + "becomes amorphous", + "containing species", + "consistent with the presence", + "peaks characteristic", + "capacity associated", + "electrodes extracted", + "against sodium metal counter", + "discharged to mv", + "above mv", + # A finite verb in the *middle*, which the boundary check cannot reach. This is what + # CLAUSE_LIKE is for, and it had only the auxiliaries. + "scan shows the xrd", + "study are open-sourced", + ], +) +def test_a_clause_is_not_a_concept(fragment: str) -> None: + assert not _is_a_term(fragment) + + +@pytest.mark.parametrize( + "term", + [ + # Ordinary terms, which must survive both lists. + "crystalline silicon", + "irreversible capacity", + "solid electrolyte interphase", + "differential capacity", + "state of charge", + "battery degradation", + "oxygen-containing yttrium hydride", + # "lead" is the metal here. A rule that loses this to catch "leads to capacity fade" is a + # bad trade, so the lead family is deliberately absent from CLAUSE_LIKE. + "lead acid battery", + "lithium lead alloy", + ], +) +def test_a_term_survives_the_verb_lists(term: str) -> None: + assert _is_a_term(term)